diff --git a/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0625/rep1/mem_samples.csv b/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0625/rep1/mem_samples.csv index 6bf40d9ef820de597b2e232fe6a21990a212293d..712d3a205eab72d8284134bde5c7b9a9bb1dfb07 100644 --- a/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0625/rep1/mem_samples.csv +++ b/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0625/rep1/mem_samples.csv @@ -14,3 +14,219 @@ 5, 0 6, 0 7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 629 +3, 629 +4, 629 +5, 629 +6, 629 +7, 629 +0, 961 +1, 949 +2, 933 +3, 957 +4, 953 +5, 929 +6, 939 +7, 947 +0, 1955 +1, 1955 +2, 1955 +3, 1955 +4, 1955 +5, 1955 +6, 1955 +7, 1955 +0, 2481 +1, 2437 +2, 2481 +3, 2459 +4, 2465 +5, 2469 +6, 2477 +7, 2467 +0, 10379 +1, 16131 +2, 10379 +3, 10379 +4, 10379 +5, 10377 +6, 10379 +7, 10379 +0, 28963 +1, 29051 +2, 28897 +3, 28897 +4, 28897 +5, 28897 +6, 29029 +7, 29051 +0, 38957 +1, 39055 +2, 38943 +3, 38901 +4, 38885 +5, 38917 +6, 39023 +7, 39089 +0, 39771 +1, 39855 +2, 39705 +3, 39705 +4, 39705 +5, 39701 +6, 39837 +7, 39855 +0, 39771 +1, 39855 +2, 39705 +3, 39705 +4, 39705 +5, 39701 +6, 39837 +7, 39855 +0, 39779 +1, 39863 +2, 39713 +3, 39713 +4, 39713 +5, 39709 +6, 39845 +7, 39863 +0, 39781 +1, 39865 +2, 39715 +3, 39715 +4, 39715 +5, 39711 +6, 39847 +7, 39865 +0, 109595 +1, 109729 +2, 109541 +3, 109553 +4, 109547 +5, 109505 +6, 110179 +7, 109725 +0, 110101 +1, 110251 +2, 110037 +3, 110035 +4, 110035 +5, 110033 +6, 110707 +7, 110251 +0, 110107 +1, 110255 +2, 110041 +3, 110039 +4, 110039 +5, 110037 +6, 110713 +7, 110255 +0, 110107 +1, 110255 +2, 110041 +3, 110039 +4, 110039 +5, 110037 +6, 110713 +7, 110255 +0, 112545 +1, 112693 +2, 112479 +3, 112477 +4, 112477 +5, 112475 +6, 111103 +7, 112693 +0, 112789 +1, 112951 +2, 112725 +3, 112733 +4, 112725 +5, 112705 +6, 111335 +7, 112937 +0, 113809 +1, 113961 +2, 113821 +3, 113831 +4, 113753 +5, 113767 +6, 112451 +7, 113973 +0, 126271 +1, 127987 +2, 127779 +3, 127777 +4, 127777 +5, 127769 +6, 126401 +7, 127987 +0, 126271 +1, 127987 +2, 127779 +3, 127777 +4, 127777 +5, 127769 +6, 126401 +7, 127987 +0, 126271 +1, 127987 +2, 127779 +3, 127777 +4, 127777 +5, 127769 +6, 126401 +7, 127987 +0, 125971 +1, 127687 +2, 127479 +3, 127477 +4, 127477 +5, 127469 +6, 126101 +7, 127687 +0, 122881 +1, 124471 +2, 124325 +3, 124067 +4, 124261 +5, 124315 +6, 122691 +7, 124471 +0, 1 +1, 4 +2, 4 +3, 132 +4, 123613 +5, 644 +6, 4 +7, 4 diff --git a/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0625/rep1/train.log b/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0625/rep1/train.log index 3fca80f6f81c6af949eebc8af841d54baa55fc58..d573c131c0bea2fb11a4705e6aa07c90e53d2884 100644 --- a/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0625/rep1/train.log +++ b/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0625/rep1/train.log @@ -11,3 +11,12739 @@ W0826 17:59:15.928000 439248 torch/distributed/run.py:766] W0826 17:59:15.928000 439248 torch/distributed/run.py:766] ***************************************** W0826 17:59:15.928000 439248 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. W0826 17:59:15.928000 439248 torch/distributed/run.py:766] ***************************************** +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 8 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.0625 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 06 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to [Gloo] Rank 74 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 23 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank [Gloo] Rank 45 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank3]:[W826 17:59:34.467740793 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 17:59:34.467849580 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 17:59:34.470143850 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 17:59:34.471749226 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 17:59:34.472986251 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 17:59:34.475082241 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 17:59:34.475122224 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.036 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 17:59:34.635753836 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.654 seconds +time to initialize megatron (seconds): 6.875 +[after megatron is initialized] datetime: 2026-08-26 17:59:40 +building GPT model ... +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4188125184 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_q_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_q_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc2.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.input_layernorm.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 17:59:43 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 17:59:43 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3700.21, 3711.07) + train/valid/test-data-iterators-setup ..........: (75.50, 88.53) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 17:59:44 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=52224 expected_recv=49446 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=52224 expected_recv=48667 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=52224 expected_recv=47653 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=52224 expected_recv=48873 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=52224 expected_recv=49446 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=52224 expected_recv=48667 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=52224 expected_recv=47653 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=52224 expected_recv=48420 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=52224 expected_recv=48420 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=52224 expected_recv=48873 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=52224 expected_recv=48121 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=52224 expected_recv=48121 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=52224 expected_recv=48558 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=52224 expected_recv=48018 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=52224 expected_recv=48558 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=52224 expected_recv=48018 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=52224 expected_recv=49529 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=52224 expected_recv=49529 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=52224 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=52224 expected_recv=49108 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=52224 expected_recv=49242 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=52224 expected_recv=49242 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=52224 expected_recv=47476 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=52224 expected_recv=48519 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=52224 expected_recv=47476 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=52224 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=48000 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=52224 expected_recv=48519 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=52224 expected_recv=49401 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=52224 expected_recv=49469 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=52224 expected_recv=49469 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=52224 expected_recv=49544 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=52224 expected_recv=49544 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=52224 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=52224 expected_recv=46521 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=52224 expected_recv=49007 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=52224 expected_recv=46521 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=47104 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=52224 expected_recv=49294 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=52224 expected_recv=47251 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=52224 expected_recv=49294 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=52224 expected_recv=47251 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=52224 expected_recv=47103 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=47872 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=52224 expected_recv=47103 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=47616 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=52224 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=52224 expected_recv=49105 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=52224 expected_recv=47831 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=52224 expected_recv=47831 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=52224 expected_recv=47521 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=52224 expected_recv=48450 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=52224 expected_recv=47521 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=52224 expected_recv=47882 expected_overflow=0 exact_recv_per_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=52224 expected_recv=48450 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=52224 expected_recv=47882 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=52224 expected_recv=48164 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=52224 expected_recv=48382 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=52224 expected_recv=48164 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=52224 expected_recv=48382 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=52224 expected_recv=47127 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=52224 expected_recv=48668 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=52224 expected_recv=47127 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=52224 expected_recv=48668 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=48000 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=52224 expected_recv=48085 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=52224 expected_recv=46684 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=52224 expected_recv=48551 expected_overflow=0 exact_recv_per_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=52224 expected_recv=48085 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=52224 expected_recv=46684 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=52224 expected_recv=48551 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=47104 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=52224 expected_recv=47266 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=52224 expected_recv=47266 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=52224 expected_recv=48787 expected_overflow=0 exact_recv_per_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=48000 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=52224 expected_recv=45979 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=52224 expected_recv=48765 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=52224 expected_recv=48787 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=52224 expected_recv=45979 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=52224 expected_recv=48765 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=52224 expected_recv=49373 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=52224 expected_recv=49373 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=52224 expected_recv=46872 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=52224 expected_recv=46872 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=52224 expected_recv=49233 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=47488 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=52224 expected_recv=49360 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=52224 expected_recv=49233 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=52224 expected_recv=49360 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=52224 expected_recv=47986 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=52224 expected_recv=47986 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=52224 expected_recv=48225 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=52224 expected_recv=48362 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=52224 expected_recv=48225 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=52224 expected_recv=48362 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=52224 expected_recv=47552 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=52224 expected_recv=47552 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=52224 expected_recv=49489 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=52224 expected_recv=49082 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=52224 expected_recv=49489 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=52224 expected_recv=49082 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=52224 expected_recv=48392 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=52224 expected_recv=48392 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=52224 expected_recv=47805 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=52224 expected_recv=47805 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=52224 expected_recv=47735 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=52224 expected_recv=47735 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=52224 expected_recv=49118 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=52224 expected_recv=47235 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=52224 expected_recv=49118 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=52224 expected_recv=47235 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=47744 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=52224 expected_recv=48645 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=52224 expected_recv=48469 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=52224 expected_recv=48645 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=52224 expected_recv=48469 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=52224 expected_recv=47459 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=52224 expected_recv=47459 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=48000 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=52224 expected_recv=46883 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=52224 expected_recv=47777 expected_overflow=0 exact_recv_per_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=52224 expected_recv=46883 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=52224 expected_recv=47777 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=47360 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=52224 expected_recv=48587 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=52224 expected_recv=48587 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=49280 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=52224 expected_recv=48910 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=52224 expected_recv=48910 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=52224 expected_recv=46765 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=52224 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=52224 expected_recv=46765 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=52224 expected_recv=48955 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=47360 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=52224 expected_recv=48353 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=52224 expected_recv=48353 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=52224 expected_recv=47617 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=52224 expected_recv=47617 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=52224 expected_recv=47894 expected_overflow=0 exact_recv_per_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=52224 expected_recv=47894 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=52224 expected_recv=47734 expected_overflow=0 exact_recv_per_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=52224 expected_recv=47734 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[53514, 54607, 44797, 53155, 44483, 47238, 50059, 45363] avg=49152.0000 max=54607 max_rank=1 min=44483 min_rank=4 imbalance_ratio=1.110982 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[49644, 49644, 48667, 49645, 49446, 47653, 49644, 48873] avg=49152.0000 max=49645 max_rank=3 min=47653 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=4962 max_load_reduction_pct=0.090867 moved_to_replicas=12759 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[44014, 59237, 50480, 48018, 42900, 39156, 60853, 48558] avg=49152.0000 max=60853 max_rank=6 min=39156 min_rank=5 imbalance_ratio=1.238057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[48420, 49644, 49644, 48018, 49644, 49643, 49645, 48558] avg=49152.0000 max=49645 max_rank=6 min=48018 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=11208 max_load_reduction_pct=0.184182 moved_to_replicas=21638 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[35968, 54922, 54737, 43717, 60867, 49529, 47882, 45594] avg=49152.0000 max=60867 max_rank=4 min=35968 min_rank=0 imbalance_ratio=1.238342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[48121, 49644, 49644, 49108, 49644, 49529, 47882, 49644] avg=49152.0000 max=49644 max_rank=1 min=47882 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=11223 max_load_reduction_pct=0.184386 moved_to_replicas=21594 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[55448, 38179, 53675, 54016, 39408, 47476, 55613, 49401] avg=49152.0000 max=55613 max_rank=6 min=38179 min_rank=1 imbalance_ratio=1.131449 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[49644, 48519, 49645, 49644, 49242, 47476, 49645, 49401] avg=49152.0000 max=49645 max_rank=2 min=47476 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=5968 max_load_reduction_pct=0.107313 moved_to_replicas=20176 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[61659, 38841, 53217, 54160, 41379, 37843, 59596, 46521] avg=49152.0000 max=61659 max_rank=0 min=37843 min_rank=5 imbalance_ratio=1.254456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[49645, 49007, 49644, 49643, 49469, 49643, 49644, 46521] avg=49152.0000 max=49645 max_rank=0 min=46521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12014 max_load_reduction_pct=0.194846 moved_to_replicas=30056 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[46868, 32261, 71096, 40491, 52321, 57507, 45421, 47251] avg=49152.0000 max=71096 max_rank=2 min=32261 min_rank=1 imbalance_ratio=1.446452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[49544, 49294, 49644, 49643, 49645, 49644, 48551, 47251] avg=49152.0000 max=49645 max_rank=4 min=47251 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21451 max_load_reduction_pct=0.301719 moved_to_replicas=31992 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[65336, 49105, 65284, 37608, 44001, 36121, 43015, 52746] avg=49152.0000 max=65336 max_rank=0 min=36121 min_rank=5 imbalance_ratio=1.329264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[49644, 49105, 49645, 49644, 47103, 49644, 48787, 49644] avg=49152.0000 max=49645 max_rank=2 min=47103 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=15691 max_load_reduction_pct=0.240159 moved_to_replicas=34434 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[38744, 50472, 39301, 60030, 62875, 46545, 50102, 45147] avg=49152.0000 max=62875 max_rank=4 min=38744 min_rank=0 imbalance_ratio=1.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[49645, 49644, 49643, 49644, 49644, 47831, 49644, 47521] avg=49152.0000 max=49645 max_rank=0 min=47521 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13230 max_load_reduction_pct=0.210417 moved_to_replicas=24903 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[52571, 38636, 59141, 45455, 36981, 57517, 51299, 51616] avg=49152.0000 max=59141 max_rank=2 min=36981 min_rank=4 imbalance_ratio=1.203227 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[49644, 48164, 49644, 48382, 48450, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=48164 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=9496 max_load_reduction_pct=0.160565 moved_to_replicas=23924 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[50932, 39002, 43736, 44820, 77617, 34416, 45471, 57222] avg=49152.0000 max=77617 max_rank=4 min=34416 min_rank=5 imbalance_ratio=1.579122 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[49644, 49644, 47127, 49644, 49644, 49644, 48225, 49644] avg=49152.0000 max=49644 max_rank=0 min=47127 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=27973 max_load_reduction_pct=0.360398 moved_to_replicas=36839 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[42774, 57074, 42872, 46684, 73428, 56602, 35363, 38419] avg=49152.0000 max=73428 max_rank=4 min=35363 min_rank=6 imbalance_ratio=1.493896 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[48668, 49644, 49644, 46684, 49644, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46684 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=23783 max_load_reduction_pct=0.323896 moved_to_replicas=38172 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[40867, 41698, 55645, 44715, 42777, 55553, 44142, 67819] avg=49152.0000 max=67819 max_rank=7 min=40867 min_rank=0 imbalance_ratio=1.379781 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[49644, 47266, 49645, 49645, 48085, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=47266 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=18174 max_load_reduction_pct=0.267978 moved_to_replicas=30085 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[27637, 57303, 45979, 29998, 64336, 58565, 63440, 45958] avg=49152.0000 max=64336 max_rank=4 min=27637 min_rank=0 imbalance_ratio=1.308919 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[49645, 49644, 45979, 49646, 49643, 49643, 49643, 49373] avg=49152.0000 max=49646 max_rank=3 min=45979 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=14690 max_load_reduction_pct=0.228333 moved_to_replicas=45068 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[25787, 46872, 54962, 38931, 34852, 72622, 65696, 53494] avg=49152.0000 max=72622 max_rank=5 min=25787 min_rank=0 imbalance_ratio=1.477498 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[48765, 46872, 49643, 49360, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=46872 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=22977 max_load_reduction_pct=0.316392 moved_to_replicas=48198 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[53546, 46101, 49233, 40849, 46802, 63686, 47777, 45222] avg=49152.0000 max=63686 max_rank=5 min=40849 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[49644, 47986, 49233, 49644, 49644, 49644, 47777, 49644] avg=49152.0000 max=49644 max_rank=0 min=47777 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=14042 max_load_reduction_pct=0.220488 moved_to_replicas=17944 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[35836, 60543, 47207, 49082, 44472, 71043, 37481, 47552] avg=49152.0000 max=71043 max_rank=5 min=35836 min_rank=0 imbalance_ratio=1.445374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[49644, 49645, 48362, 49082, 49644, 49644, 49643, 47552] avg=49152.0000 max=49645 max_rank=1 min=47552 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=21398 max_load_reduction_pct=0.301198 moved_to_replicas=32298 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[81979, 45882, 49489, 35435, 51567, 34928, 45544, 48392] avg=49152.0000 max=81979 max_rank=0 min=34928 min_rank=5 imbalance_ratio=1.667867 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[49643, 47805, 49489, 49643, 49644, 49645, 48955, 48392] avg=49152.0000 max=49645 max_rank=5 min=47805 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=32334 max_load_reduction_pct=0.394418 moved_to_replicas=34258 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[44846, 84497, 57001, 38357, 53893, 43846, 39486, 31290] avg=49152.0000 max=84497 max_rank=1 min=31290 min_rank=7 imbalance_ratio=1.719096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49645, 47617, 47735] avg=49152.0000 max=49645 max_rank=5 min=47617 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=34852 max_load_reduction_pct=0.412464 moved_to_replicas=46459 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[53351, 41444, 45654, 53107, 40708, 32224, 75597, 51131] avg=49152.0000 max=75597 max_rank=6 min=32224 min_rank=5 imbalance_ratio=1.538025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[49644, 48645, 49118, 49643, 49644, 47235, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47235 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=25953 max_load_reduction_pct=0.343307 moved_to_replicas=34610 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[40115, 44373, 57669, 63557, 29035, 66245, 52477, 39745] avg=49152.0000 max=66245 max_rank=5 min=29035 min_rank=4 imbalance_ratio=1.347758 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[49642, 46883, 49645, 49644, 48469, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=46883 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=16600 max_load_reduction_pct=0.250585 moved_to_replicas=41372 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[37595, 60365, 82329, 42400, 65564, 30276, 34987, 39700] avg=49152.0000 max=82329 max_rank=2 min=30276 min_rank=5 imbalance_ratio=1.674988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[47459, 49644, 49645, 49643, 49644, 49643, 47894, 49644] avg=49152.0000 max=49645 max_rank=2 min=47459 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=32684 max_load_reduction_pct=0.396993 moved_to_replicas=59326 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[52147, 44015, 60436, 58110, 32733, 43141, 58705, 43929] avg=49152.0000 max=60436 max_rank=2 min=32733 min_rank=4 imbalance_ratio=1.229574 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[49643, 46765, 49644, 49645, 48587, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=46765 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=10791 max_load_reduction_pct=0.178553 moved_to_replicas=30822 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=23 total_expected_recv=1133511 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55552 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[33689, 55359, 61244, 64202, 53265, 56739, 40638, 28080] avg=49152.0000 max=64202 max_rank=3 min=28080 min_rank=7 imbalance_ratio=1.306193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[48910, 49645, 49644, 49643, 49644, 49643, 47734, 48353] avg=49152.0000 max=49645 max_rank=1 min=47734 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=14557 max_load_reduction_pct=0.226737 moved_to_replicas=42589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=23 total_expected_recv=1134190 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=53632 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=23 total_expected_recv=1133313 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55296 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=23 total_expected_recv=1132282 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56448 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=23 total_expected_recv=1125260 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=61184 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=23 total_expected_recv=1122397 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=65152 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=23 total_expected_recv=1134581 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=53504 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=23 total_expected_recv=1128434 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59520 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 18:00:13] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 29649.8 | throughput per GPU (TFLOP/s/GPU): 4.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092127E+01 | loss scale: 1.0 | grad norm: 30.623 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +Number of parameters in transformer block in billions: 13.53 +Number of parameters in embedding layers in billions: 0.21 +Total number of parameters in billions: 13.74 +Number of parameters in most loaded shard in billions: 13.7356 +Theoretical memory footprints: weight and optimizer=98244.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 37116.89306640625 | max allocated: 102645.49365234375 | reserved: 105678.0 | max reserved: 105678.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=52224 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=52224 expected_recv=47725 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=52224 expected_recv=48091 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=52224 expected_recv=49181 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=52224 expected_recv=47725 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=52224 expected_recv=48091 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=52224 expected_recv=47977 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=52224 expected_recv=47977 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=52224 expected_recv=47820 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=52224 expected_recv=49199 expected_overflow=0 exact_recv_per_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=52224 expected_recv=47820 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=52224 expected_recv=49199 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=52224 expected_recv=47761 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=52224 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=52224 expected_recv=47761 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=52224 expected_recv=49138 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=52224 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=52224 expected_recv=48763 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=52224 expected_recv=49227 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=52224 expected_recv=48763 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=52224 expected_recv=49395 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=52224 expected_recv=49395 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=52224 expected_recv=48529 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=52224 expected_recv=48697 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=52224 expected_recv=48529 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=52224 expected_recv=48697 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49280 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=52224 expected_recv=48965 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=52224 expected_recv=48965 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=52224 expected_recv=48451 expected_overflow=0 exact_recv_per_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=52224 expected_recv=48451 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=52224 expected_recv=48708 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=52224 expected_recv=48708 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=52224 expected_recv=47048 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=52224 expected_recv=47048 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=52224 expected_recv=49240 expected_overflow=0 exact_recv_per_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=47616 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=52224 expected_recv=49240 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=52224 expected_recv=48060 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=52224 expected_recv=47292 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=52224 expected_recv=48060 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=52224 expected_recv=47292 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=47616 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=52224 expected_recv=47804 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=52224 expected_recv=47967 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=52224 expected_recv=47804 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=52224 expected_recv=47967 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=52224 expected_recv=49224 expected_overflow=0 exact_recv_per_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=52224 expected_recv=47712 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=52224 expected_recv=49224 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=52224 expected_recv=47712 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=52224 expected_recv=48762 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=52224 expected_recv=48762 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=52224 expected_recv=48522 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=52224 expected_recv=48522 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=52224 expected_recv=47632 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=52224 expected_recv=47632 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=52224 expected_recv=49504 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=52224 expected_recv=47860 expected_overflow=0 exact_recv_per_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=52224 expected_recv=49504 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=52224 expected_recv=47850 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=52224 expected_recv=47860 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=52224 expected_recv=47850 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=52224 expected_recv=47615 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=52224 expected_recv=47615 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=52224 expected_recv=49530 expected_overflow=0 exact_recv_per_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=52224 expected_recv=48346 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=52224 expected_recv=49530 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=52224 expected_recv=48346 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=52224 expected_recv=47005 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=52224 expected_recv=47005 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=47360 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=52224 expected_recv=48500 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=52224 expected_recv=49063 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=52224 expected_recv=48500 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=52224 expected_recv=49063 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=52224 expected_recv=48200 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=52224 expected_recv=48877 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=52224 expected_recv=48200 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=52224 expected_recv=48877 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=52224 expected_recv=47920 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=52224 expected_recv=48055 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=52224 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=52224 expected_recv=47920 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=52224 expected_recv=48055 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=52224 expected_recv=49019 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=52224 expected_recv=46896 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=52224 expected_recv=46896 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=47744 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=52224 expected_recv=48456 expected_overflow=0 exact_recv_per_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=52224 expected_recv=48456 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=52224 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=52224 expected_recv=49149 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=52224 expected_recv=49046 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=52224 expected_recv=47125 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=52224 expected_recv=49046 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=52224 expected_recv=47125 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=47744 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=52224 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=52224 expected_recv=48405 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=52224 expected_recv=49319 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=52224 expected_recv=48405 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=52224 expected_recv=46948 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=52224 expected_recv=46948 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=47744 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=52224 expected_recv=49325 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=52224 expected_recv=49325 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=52224 expected_recv=47820 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=52224 expected_recv=47851 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=52224 expected_recv=47820 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=52224 expected_recv=47851 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=52224 expected_recv=49112 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=52224 expected_recv=48769 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=52224 expected_recv=47116 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=52224 expected_recv=49112 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=52224 expected_recv=48769 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=52224 expected_recv=47116 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=47872 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=52224 expected_recv=49427 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=52224 expected_recv=49427 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=52224 expected_recv=46719 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=52224 expected_recv=48851 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=52224 expected_recv=46719 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=52224 expected_recv=48851 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=47360 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=52224 expected_recv=46024 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=52224 expected_recv=46024 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=46592 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=52224 expected_recv=49329 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=52224 expected_recv=49329 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=52224 expected_recv=47601 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=52224 expected_recv=47601 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=48000 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=52224 expected_recv=47751 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=52224 expected_recv=47751 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=52224 expected_recv=46982 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=52224 expected_recv=48369 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=52224 expected_recv=46982 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=52224 expected_recv=48369 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=47872 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=52224 expected_recv=49497 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=52224 expected_recv=49497 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=52224 expected_recv=47561 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=52224 expected_recv=47561 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=52224 expected_recv=47938 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=52224 expected_recv=47938 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[54846, 55191, 43634, 52605, 44694, 46305, 51063, 44878] avg=49152.0000 max=55191 max_rank=1 min=43634 min_rank=2 imbalance_ratio=1.122864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[49644, 49644, 49181, 49644, 49644, 47725, 49643, 48091] avg=49152.0000 max=49644 max_rank=0 min=47725 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=5547 max_load_reduction_pct=0.100506 moved_to_replicas=15129 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[43094, 58796, 51904, 47820, 43163, 38850, 61612, 47977] avg=49152.0000 max=61612 max_rank=6 min=38850 min_rank=5 imbalance_ratio=1.253499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[49199, 49644, 49644, 47820, 49644, 49644, 49644, 47977] avg=49152.0000 max=49644 max_rank=1 min=47820 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=11968 max_load_reduction_pct=0.194248 moved_to_replicas=23380 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[37808, 53269, 54263, 42045, 63330, 49138, 47761, 45602] avg=49152.0000 max=63330 max_rank=4 min=37808 min_rank=0 imbalance_ratio=1.288452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[49395, 49644, 49645, 48763, 49643, 49138, 47761, 49227] avg=49152.0000 max=49645 max_rank=2 min=47761 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=13685 max_load_reduction_pct=0.216090 moved_to_replicas=21930 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[57134, 38359, 54935, 52422, 38260, 48697, 54444, 48965] avg=49152.0000 max=57134 max_rank=0 min=38260 min_rank=4 imbalance_ratio=1.162394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[49643, 48451, 49643, 49644, 48529, 48697, 49644, 48965] avg=49152.0000 max=49644 max_rank=3 min=48451 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=7490 max_load_reduction_pct=0.131095 moved_to_replicas=20359 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[63425, 42308, 55663, 52221, 41818, 36735, 56575, 44471] avg=49152.0000 max=63425 max_rank=0 min=36735 min_rank=5 imbalance_ratio=1.290385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[49645, 49240, 49645, 49644, 48708, 49643, 49643, 47048] avg=49152.0000 max=49645 max_rank=0 min=47048 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=13780 max_load_reduction_pct=0.217264 moved_to_replicas=29308 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[43076, 34000, 73870, 41111, 50750, 60747, 42708, 46954] avg=49152.0000 max=73870 max_rank=2 min=34000 min_rank=1 imbalance_ratio=1.502889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[47292, 49644, 49644, 49644, 49644, 49644, 49644, 48060] avg=49152.0000 max=49644 max_rank=1 min=47292 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=24226 max_load_reduction_pct=0.327955 moved_to_replicas=36435 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[65746, 49224, 68571, 37409, 43902, 32555, 39042, 56767] avg=49152.0000 max=68571 max_rank=2 min=32555 min_rank=5 imbalance_ratio=1.395081 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[49644, 49224, 49645, 49643, 47967, 47804, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47804 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=18926 max_load_reduction_pct=0.276006 moved_to_replicas=42152 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[36173, 50575, 40709, 65199, 63406, 42591, 47468, 47095] avg=49152.0000 max=65199 max_rank=3 min=36173 min_rank=0 imbalance_ratio=1.326477 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[48522, 49644, 49643, 49644, 49644, 49645, 47712, 48762] avg=49152.0000 max=49645 max_rank=5 min=47712 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=15554 max_load_reduction_pct=0.238562 moved_to_replicas=30248 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[49504, 37783, 55749, 49855, 39118, 53444, 51843, 55920] avg=49152.0000 max=55920 max_rank=7 min=37783 min_rank=1 imbalance_ratio=1.137695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[49504, 47860, 49644, 49644, 47632, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=47632 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=6275 max_load_reduction_pct=0.112214 moved_to_replicas=18591 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[60400, 35085, 50740, 46519, 70426, 33972, 40311, 55763] avg=49152.0000 max=70426 max_rank=4 min=33972 min_rank=5 imbalance_ratio=1.432821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[49644, 49530, 49644, 47615, 49644, 47850, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47615 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20781 max_load_reduction_pct=0.295076 moved_to_replicas=38753 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[51213, 55147, 45668, 47005, 70501, 55673, 33430, 34579] avg=49152.0000 max=70501 max_rank=4 min=33430 min_rank=6 imbalance_ratio=1.434347 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[49645, 49644, 48346, 47005, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=0 min=47005 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=20856 max_load_reduction_pct=0.295826 moved_to_replicas=33958 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[43758, 41449, 48500, 46908, 49063, 52839, 47058, 63641] avg=49152.0000 max=63641 max_rank=7 min=41449 min_rank=1 imbalance_ratio=1.294779 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[48877, 49644, 48500, 48200, 49063, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=48200 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=13997 max_load_reduction_pct=0.219937 moved_to_replicas=17192 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[28124, 55507, 53276, 31759, 64677, 53624, 61962, 44287] avg=49152.0000 max=64677 max_rank=4 min=28124 min_rank=0 imbalance_ratio=1.315857 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[49019, 49644, 49643, 48055, 49645, 49645, 49645, 47920] avg=49152.0000 max=49645 max_rank=4 min=47920 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=15032 max_load_reduction_pct=0.232416 moved_to_replicas=40826 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[28101, 41601, 52917, 33620, 33104, 79255, 74422, 50196] avg=49152.0000 max=79255 max_rank=5 min=28101 min_rank=0 imbalance_ratio=1.612447 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[49644, 48456, 49644, 49644, 46896, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46896 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=29611 max_load_reduction_pct=0.373617 moved_to_replicas=58214 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[52394, 45754, 45404, 41445, 53209, 59564, 46400, 49046] avg=49152.0000 max=59564 max_rank=5 min=41445 min_rank=3 imbalance_ratio=1.211833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[49645, 49319, 47125, 49645, 49644, 49643, 49149, 49046] avg=49152.0000 max=49645 max_rank=0 min=47125 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=9919 max_load_reduction_pct=0.166527 moved_to_replicas=16235 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[32405, 58749, 44681, 45376, 56178, 59923, 44647, 51257] avg=49152.0000 max=59923 max_rank=5 min=32405 min_rank=0 imbalance_ratio=1.219137 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[49645, 49644, 49642, 46948, 49645, 49643, 48405, 49644] avg=49152.0000 max=49645 max_rank=0 min=46948 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=10278 max_load_reduction_pct=0.171520 moved_to_replicas=27531 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[88984, 45318, 47746, 37220, 49325, 34912, 42712, 46999] avg=49152.0000 max=88984 max_rank=0 min=34912 min_rank=5 imbalance_ratio=1.810384 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[49644, 49645, 47851, 49642, 49325, 49645, 49644, 47820] avg=49152.0000 max=49645 max_rank=1 min=47820 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=39339 max_load_reduction_pct=0.442091 moved_to_replicas=39340 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[43337, 83253, 47109, 40249, 62099, 44576, 38162, 34431] avg=49152.0000 max=83253 max_rank=1 min=34431 min_rank=7 imbalance_ratio=1.693787 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[47116, 49644, 48769, 49644, 49643, 49112, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47116 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33609 max_load_reduction_pct=0.403697 moved_to_replicas=46064 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[45380, 35743, 41956, 54109, 49427, 39084, 74402, 53115] avg=49152.0000 max=74402 max_rank=6 min=35743 min_rank=1 imbalance_ratio=1.513713 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[48851, 49644, 46719, 49643, 49427, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=46719 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24758 max_load_reduction_pct=0.332760 moved_to_replicas=32694 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[31805, 60309, 61056, 44954, 33984, 80801, 43354, 36953] avg=49152.0000 max=80801 max_rank=5 min=31805 min_rank=0 imbalance_ratio=1.643901 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[49644, 49644, 49643, 49329, 46024, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=46024 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=31156 max_load_reduction_pct=0.385589 moved_to_replicas=53234 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[43312, 53980, 79365, 56248, 59249, 21033, 37116, 42913] avg=49152.0000 max=79365 max_rank=2 min=21033 min_rank=5 imbalance_ratio=1.614685 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[47751, 49644, 49644, 49644, 49643, 49645, 47601, 49644] avg=49152.0000 max=49645 max_rank=5 min=47601 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29720 max_load_reduction_pct=0.374472 moved_to_replicas=50266 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[49747, 43490, 48369, 57408, 34093, 52750, 62102, 45257] avg=49152.0000 max=62102 max_rank=6 min=34093 min_rank=4 imbalance_ratio=1.263468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[49644, 49645, 48369, 49644, 49643, 49644, 49645, 46982] avg=49152.0000 max=49645 max_rank=1 min=46982 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=12457 max_load_reduction_pct=0.200589 moved_to_replicas=23431 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[46186, 60860, 44643, 54757, 49497, 51283, 58767, 27223] avg=49152.0000 max=60860 max_rank=1 min=27223 min_rank=7 imbalance_ratio=1.238200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[47938, 49645, 49645, 49643, 49497, 49643, 49644, 47561] avg=49152.0000 max=49645 max_rank=1 min=47561 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=11215 max_load_reduction_pct=0.184275 moved_to_replicas=27091 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=23 total_expected_recv=1134221 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55040 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=23 total_expected_recv=1128437 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59520 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=23 total_expected_recv=1123546 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=64128 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=23 total_expected_recv=1134273 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=54656 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=23 total_expected_recv=1136387 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=51840 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=23 total_expected_recv=1129518 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58880 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=23 total_expected_recv=1129195 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58496 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=23 total_expected_recv=1128391 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59264 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 18:00:14] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 641.9 | throughput per GPU (TFLOP/s/GPU): 198.8 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091995E+01 | loss scale: 1.0 | grad norm: 30.567 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=52224 expected_recv=47905 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=52224 expected_recv=48827 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=52224 expected_recv=48263 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=52224 expected_recv=47905 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=52224 expected_recv=48827 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=52224 expected_recv=48263 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=52224 expected_recv=48217 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=52224 expected_recv=47135 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=52224 expected_recv=48217 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=52224 expected_recv=47135 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=47616 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=52224 expected_recv=48651 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=52224 expected_recv=47656 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=52224 expected_recv=48651 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=52224 expected_recv=47656 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=49280 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=52224 expected_recv=48686 expected_overflow=0 exact_recv_per_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=52224 expected_recv=47875 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=52224 expected_recv=48686 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=52224 expected_recv=47875 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=52224 expected_recv=48531 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=52224 expected_recv=48589 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=52224 expected_recv=48531 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=52224 expected_recv=48589 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49280 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=52224 expected_recv=47592 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=52224 expected_recv=49526 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=52224 expected_recv=47878 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=52224 expected_recv=47592 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=52224 expected_recv=49526 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=52224 expected_recv=47878 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=52224 expected_recv=47867 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=52224 expected_recv=47867 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=52224 expected_recv=47484 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=52224 expected_recv=47484 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=52224 expected_recv=48019 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=52224 expected_recv=47333 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=52224 expected_recv=48019 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=52224 expected_recv=47333 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=52224 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=52224 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=52224 expected_recv=47049 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=52224 expected_recv=48303 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=52224 expected_recv=47049 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=52224 expected_recv=48303 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=47744 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=52224 expected_recv=47323 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=52224 expected_recv=48201 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=52224 expected_recv=49472 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=52224 expected_recv=47323 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=52224 expected_recv=48201 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=52224 expected_recv=49472 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=52224 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=52224 expected_recv=45709 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=52224 expected_recv=48241 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=52224 expected_recv=47110 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=52224 expected_recv=48241 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=52224 expected_recv=47110 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=52224 expected_recv=46290 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=47744 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=52224 expected_recv=46290 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=47104 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=52224 expected_recv=49062 expected_overflow=0 exact_recv_per_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=52224 expected_recv=49062 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=46208 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=52224 expected_recv=46959 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=52224 expected_recv=48392 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=52224 expected_recv=46959 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=52224 expected_recv=48392 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=47616 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=52224 expected_recv=49580 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=52224 expected_recv=49639 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=52224 expected_recv=49580 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=52224 expected_recv=49639 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=52224 expected_recv=48855 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=52224 expected_recv=48855 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=52224 expected_recv=46562 expected_overflow=0 exact_recv_per_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=52224 expected_recv=46562 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=47104 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=52224 expected_recv=45713 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=52224 expected_recv=45713 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=46208 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=52224 expected_recv=48362 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=52224 expected_recv=48362 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=52224 expected_recv=46991 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=52224 expected_recv=46991 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=47872 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=52224 expected_recv=46596 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=52224 expected_recv=46596 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=52224 expected_recv=48756 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=47488 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=52224 expected_recv=48756 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=52224 expected_recv=47358 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=52224 expected_recv=47358 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=47872 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=46848 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=52224 expected_recv=47993 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=52224 expected_recv=47993 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[50266, 51688, 50006, 44588, 44688, 48827, 55252, 47901] avg=49152.0000 max=55252 max_rank=6 min=44588 min_rank=3 imbalance_ratio=1.124105 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 47905, 48827, 49644, 48263] avg=49152.0000 max=49645 max_rank=1 min=47905 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=5607 max_load_reduction_pct=0.101480 moved_to_replicas=8636 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[40601, 54729, 60977, 50778, 34784, 37481, 67865, 46001] avg=49152.0000 max=67865 max_rank=6 min=34784 min_rank=4 imbalance_ratio=1.380717 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 48217, 49644, 47135] avg=49152.0000 max=49644 max_rank=0 min=47135 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18221 max_load_reduction_pct=0.268489 moved_to_replicas=35773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[34918, 50618, 52477, 63127, 75594, 42284, 33639, 40559] avg=49152.0000 max=75594 max_rank=4 min=33639 min_rank=6 imbalance_ratio=1.537964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[48686, 49644, 49645, 49644, 49646, 47656, 48651, 49644] avg=49152.0000 max=49646 max_rank=4 min=47656 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=25948 max_load_reduction_pct=0.343255 moved_to_replicas=43240 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[60897, 39353, 60405, 58880, 34670, 52088, 55188, 31735] avg=49152.0000 max=60897 max_rank=0 min=31735 min_rank=7 imbalance_ratio=1.238953 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[49644, 48589, 49644, 49644, 47875, 49644, 49645, 48531] avg=49152.0000 max=49645 max_rank=6 min=47875 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=11252 max_load_reduction_pct=0.184771 moved_to_replicas=39238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[44781, 47878, 69828, 74940, 31256, 49526, 45893, 29114] avg=49152.0000 max=74940 max_rank=3 min=29114 min_rank=7 imbalance_ratio=1.524658 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[49643, 47878, 49645, 49645, 49643, 49526, 49644, 47592] avg=49152.0000 max=49645 max_rank=2 min=47592 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=25295 max_load_reduction_pct=0.337537 moved_to_replicas=45480 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[58421, 47867, 78872, 30595, 64030, 32012, 56026, 25393] avg=49152.0000 max=78872 max_rank=2 min=25393 min_rank=7 imbalance_ratio=1.604655 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[49644, 47867, 49644, 47484, 49645, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=47484 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=29227 max_load_reduction_pct=0.370562 moved_to_replicas=58773 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[65053, 86962, 34580, 21589, 22204, 105107, 48019, 9702] avg=49152.0000 max=105107 max_rank=5 min=9702 min_rank=7 imbalance_ratio=2.138407 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[49644, 49644, 47333, 49644, 49644, 49644, 48019, 49644] avg=49152.0000 max=49644 max_rank=0 min=47333 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=55463 max_load_reduction_pct=0.527681 moved_to_replicas=108190 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[38655, 71070, 32312, 98453, 32636, 26446, 59933, 33711] avg=49152.0000 max=98453 max_rank=3 min=26446 min_rank=5 imbalance_ratio=2.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[45707, 49645, 49644, 49644, 49645, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=48808 max_load_reduction_pct=0.495749 moved_to_replicas=80524 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[67939, 28256, 16639, 68436, 40543, 78277, 15497, 77629] avg=49152.0000 max=78277 max_rank=5 min=15497 min_rank=6 imbalance_ratio=1.592550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[49644, 47049, 48303, 49643, 49644, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47049 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28632 max_load_reduction_pct=0.365778 moved_to_replicas=93705 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[39071, 18384, 24821, 124013, 41738, 24713, 71004, 49472] avg=49152.0000 max=124013 max_rank=3 min=18384 min_rank=1 imbalance_ratio=2.523051 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[49645, 48201, 47323, 49644, 49643, 49644, 49644, 49472] avg=49152.0000 max=49645 max_rank=0 min=47323 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=74368 max_load_reduction_pct=0.599679 moved_to_replicas=95729 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[63094, 45124, 28349, 72270, 17140, 85957, 54642, 26640] avg=49152.0000 max=85957 max_rank=5 min=17140 min_rank=4 imbalance_ratio=1.748800 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49644, 49644, 49644, 45709] avg=49152.0000 max=49644 max_rank=1 min=45709 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=36313 max_load_reduction_pct=0.422455 moved_to_replicas=77387 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[61389, 25899, 41270, 32334, 60559, 13764, 110891, 47110] avg=49152.0000 max=110891 max_rank=6 min=13764 min_rank=5 imbalance_ratio=2.256083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[49643, 48241, 49645, 49645, 49644, 49644, 49644, 47110] avg=49152.0000 max=49645 max_rank=2 min=47110 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61246 max_load_reduction_pct=0.552308 moved_to_replicas=83907 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[16202, 62030, 83119, 61063, 22180, 70898, 40391, 37333] avg=49152.0000 max=83119 max_rank=2 min=16202 min_rank=0 imbalance_ratio=1.691060 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[49062, 49645, 49644, 49644, 46290, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=46290 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=33474 max_load_reduction_pct=0.402724 moved_to_replicas=78534 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[43395, 4716, 97309, 67761, 23730, 29976, 88325, 38004] avg=49152.0000 max=97309 max_rank=2 min=4716 min_rank=1 imbalance_ratio=1.979757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=47664 max_load_reduction_pct=0.489821 moved_to_replicas=104463 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[11766, 66549, 20355, 55391, 23515, 145281, 32718, 37641] avg=49152.0000 max=145281 max_rank=5 min=11766 min_rank=0 imbalance_ratio=2.955750 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[49643, 49644, 49645, 49644, 49644, 49645, 48392, 46959] avg=49152.0000 max=49645 max_rank=2 min=46959 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=95636 max_load_reduction_pct=0.658283 moved_to_replicas=118289 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[64012, 108068, 23318, 24568, 8085, 53472, 29956, 81737] avg=49152.0000 max=108068 max_rank=1 min=8085 min_rank=4 imbalance_ratio=2.198649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=58423 max_load_reduction_pct=0.540613 moved_to_replicas=108713 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[54334, 49580, 22768, 16769, 60089, 44165, 68505, 77006] avg=49152.0000 max=77006 max_rank=7 min=16769 min_rank=3 imbalance_ratio=1.566691 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[49644, 49580, 49644, 46562, 49644, 48855, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46562 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=27362 max_load_reduction_pct=0.355323 moved_to_replicas=61358 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[13946, 97793, 75155, 54399, 43589, 45713, 40178, 22443] avg=49152.0000 max=97793 max_rank=1 min=13946 min_rank=0 imbalance_ratio=1.989604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49639, 45713, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=45713 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48148 max_load_reduction_pct=0.492346 moved_to_replicas=78415 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[24064, 42006, 52164, 84155, 21001, 1729, 94610, 73487] avg=49152.0000 max=94610 max_rank=6 min=1729 min_rank=5 imbalance_ratio=1.924845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[49644, 48362, 49644, 49644, 49644, 46991, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=46991 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=44966 max_load_reduction_pct=0.475277 moved_to_replicas=105840 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[19997, 56222, 58089, 95031, 9649, 51884, 40017, 62327] avg=49152.0000 max=95031 max_rank=3 min=9649 min_rank=4 imbalance_ratio=1.933411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[48756, 49643, 49644, 49644, 49645, 49644, 46596, 49644] avg=49152.0000 max=49645 max_rank=4 min=46596 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=45386 max_load_reduction_pct=0.477592 moved_to_replicas=75333 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[65642, 17890, 36100, 21901, 129137, 68550, 4062, 49934] avg=49152.0000 max=129137 max_rank=4 min=4062 min_rank=6 imbalance_ratio=2.627299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[49644, 49642, 49644, 45708, 49646, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=79491 max_load_reduction_pct=0.615556 moved_to_replicas=114687 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[65502, 36242, 64919, 1897, 31501, 76057, 107217, 9881] avg=49152.0000 max=107217 max_rank=6 min=1897 min_rank=3 imbalance_ratio=2.181335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 47358, 49644, 49644, 47993] avg=49152.0000 max=49645 max_rank=0 min=47358 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=57572 max_load_reduction_pct=0.536967 moved_to_replicas=115119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[54042, 32430, 62638, 22394, 20564, 64697, 79600, 56851] avg=49152.0000 max=79600 max_rank=6 min=20564 min_rank=4 imbalance_ratio=1.619466 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=29956 max_load_reduction_pct=0.376332 moved_to_replicas=69608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=23 total_expected_recv=1132664 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=54016 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=23 total_expected_recv=1134893 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=51712 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=23 total_expected_recv=1131906 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55168 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=23 total_expected_recv=1130427 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57600 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=23 total_expected_recv=1123780 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=63232 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=23 total_expected_recv=1135446 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=52352 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=23 total_expected_recv=1126154 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=61184 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=23 total_expected_recv=1128698 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58112 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 18:00:14] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 635.3 | throughput per GPU (TFLOP/s/GPU): 200.8 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.952354E+00 | loss scale: 1.0 | grad norm: 42.232 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=52224 expected_recv=48777 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=52224 expected_recv=47005 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=52224 expected_recv=49215 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=52224 expected_recv=48777 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=52224 expected_recv=47005 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=47744 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=52224 expected_recv=49215 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=52224 expected_recv=46897 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=52224 expected_recv=48940 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=52224 expected_recv=49427 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=52224 expected_recv=46897 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=52224 expected_recv=48940 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=47488 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=52224 expected_recv=49427 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=52224 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=52224 expected_recv=49377 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=52224 expected_recv=47626 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=52224 expected_recv=47725 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=52224 expected_recv=47626 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=52224 expected_recv=47725 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=52224 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=52224 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=52224 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=52224 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=52224 expected_recv=49521 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=52224 expected_recv=49521 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=52224 expected_recv=45832 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=52224 expected_recv=45832 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=52224 expected_recv=46144 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=52224 expected_recv=46144 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=46848 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=52224 expected_recv=49208 expected_overflow=0 exact_recv_per_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=52224 expected_recv=49208 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=52224 expected_recv=47562 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=52224 expected_recv=47241 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=52224 expected_recv=48110 expected_overflow=0 exact_recv_per_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=52224 expected_recv=47562 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=52224 expected_recv=47241 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=52224 expected_recv=48110 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=47616 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=52224 expected_recv=49280 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=52224 expected_recv=48153 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=52224 expected_recv=49280 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=52224 expected_recv=48153 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=52224 expected_recv=47177 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=52224 expected_recv=47177 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=47360 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=52224 expected_recv=48176 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=52224 expected_recv=46417 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=52224 expected_recv=48176 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=52224 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=52224 expected_recv=46417 actual_valid_recv=46976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=52224 expected_recv=48936 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=46976 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=52224 expected_recv=47588 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=52224 expected_recv=47588 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=47872 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=52224 expected_recv=48142 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=52224 expected_recv=49266 expected_overflow=0 exact_recv_per_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=52224 expected_recv=48142 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=45952 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=52224 expected_recv=49266 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=45952 slack_slots=6272 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=52224 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=52224 expected_recv=45707 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=46208 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=52224 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=52224 expected_recv=45709 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=46208 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=52224 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=52224 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=52224 expected_recv=48465 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=52224 expected_recv=48465 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=49280 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=52224 expected_recv=46885 expected_overflow=0 exact_recv_per_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=52224 expected_recv=46885 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=47360 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=52224 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=46080 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=52224 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=52224 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=52224 expected_recv=45707 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=46080 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[56824, 56034, 43327, 42034, 55878, 43352, 48777, 46990] avg=49152.0000 max=56824 max_rank=0 min=42034 min_rank=3 imbalance_ratio=1.156087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[49643, 49645, 49643, 49215, 49643, 49645, 48777, 47005] avg=49152.0000 max=49645 max_rank=1 min=47005 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=7179 max_load_reduction_pct=0.126337 moved_to_replicas=19804 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 60901, 62570, 11587, 32383, 84954, 42504] avg=49152.0000 max=84954 max_rank=6 min=11587 min_rank=4 imbalance_ratio=1.728394 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[49377, 48940, 49644, 49643, 46897, 49427, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=46897 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=35310 max_load_reduction_pct=0.415637 moved_to_replicas=59493 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[58796, 38431, 87782, 68748, 58839, 33764, 20864, 25992] avg=49152.0000 max=87782 max_rank=2 min=20864 min_rank=6 imbalance_ratio=1.785929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[49644, 47626, 49644, 49644, 49644, 49644, 49645, 47725] avg=49152.0000 max=49645 max_rank=6 min=47626 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=38137 max_load_reduction_pct=0.434451 moved_to_replicas=75589 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[35441, 14326, 84044, 101452, 34513, 51443, 39098, 32899] avg=49152.0000 max=101452 max_rank=3 min=14326 min_rank=1 imbalance_ratio=2.064046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[49644, 49645, 49644, 49644, 49643, 49644, 49645, 45707] avg=49152.0000 max=49645 max_rank=1 min=45707 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=51807 max_load_reduction_pct=0.510655 moved_to_replicas=88007 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[18405, 11442, 81114, 91757, 86369, 32173, 34064, 37892] avg=49152.0000 max=91757 max_rank=3 min=11442 min_rank=1 imbalance_ratio=1.866801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 45708, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=42113 max_load_reduction_pct=0.458962 moved_to_replicas=110308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[25095, 40926, 183772, 3179, 27734, 71706, 34507, 6297] avg=49152.0000 max=183772 max_rank=2 min=3179 min_rank=3 imbalance_ratio=3.738851 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[49645, 49645, 49644, 49644, 45707, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=134127 max_load_reduction_pct=0.729855 moved_to_replicas=156190 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[112295, 127799, 26414, 16815, 42929, 21722, 44736, 506] avg=49152.0000 max=127799 max_rank=1 min=506 min_rank=7 imbalance_ratio=2.600077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[49643, 49644, 49643, 45832, 49644, 49645, 49521, 49644] avg=49152.0000 max=49645 max_rank=5 min=45832 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=78154 max_load_reduction_pct=0.611538 moved_to_replicas=140806 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[14355, 140497, 9232, 79284, 70476, 23923, 46144, 9305] avg=49152.0000 max=140497 max_rank=1 min=9232 min_rank=2 imbalance_ratio=2.858419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[49208, 49644, 49644, 49643, 49643, 49645, 46144, 49645] avg=49152.0000 max=49645 max_rank=5 min=46144 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=90852 max_load_reduction_pct=0.646647 moved_to_replicas=141325 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[39297, 17198, 5586, 47241, 7781, 117653, 25376, 133084] avg=49152.0000 max=133084 max_rank=7 min=5586 min_rank=2 imbalance_ratio=2.707601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[48110, 49645, 49644, 47241, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=47241 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=83439 max_load_reduction_pct=0.626965 moved_to_replicas=151449 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[133438, 101, 33693, 109057, 402, 48153, 49280, 19092] avg=49152.0000 max=133438 max_rank=0 min=101 min_rank=1 imbalance_ratio=2.714803 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[49643, 49644, 49646, 49644, 47562, 48153, 49280, 49644] avg=49152.0000 max=49646 max_rank=2 min=47562 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=83792 max_load_reduction_pct=0.627947 moved_to_replicas=143207 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[81101, 61575, 13290, 48176, 6637, 143540, 31744, 7153] avg=49152.0000 max=143540 max_rank=5 min=6637 min_rank=4 imbalance_ratio=2.920329 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[49644, 49644, 49644, 48176, 49645, 49644, 47177, 49642] avg=49152.0000 max=49645 max_rank=4 min=47177 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=93895 max_load_reduction_pct=0.654138 moved_to_replicas=137284 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[63851, 69246, 24463, 79440, 11823, 48936, 16554, 78903] avg=49152.0000 max=79440 max_rank=3 min=11823 min_rank=4 imbalance_ratio=1.616211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 49645, 48936, 46417, 49643] avg=49152.0000 max=49645 max_rank=4 min=46417 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=29795 max_load_reduction_pct=0.375063 moved_to_replicas=92864 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[2, 39736, 99658, 22793, 31063, 124591, 62164, 13209] avg=49152.0000 max=124591 max_rank=5 min=2 min_rank=0 imbalance_ratio=2.534810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=74946 max_load_reduction_pct=0.601536 moved_to_replicas=137481 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[3, 38652, 55223, 42563, 64376, 60899, 58580, 72920] avg=49152.0000 max=72920 max_rank=7 min=3 min_rank=0 imbalance_ratio=1.483561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[49266, 47588, 49644, 48142, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47588 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=23276 max_load_reduction_pct=0.319199 moved_to_replicas=63778 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[4326, 41857, 119107, 57319, 32267, 88258, 44869, 5213] avg=49152.0000 max=119107 max_rank=2 min=4326 min_rank=0 imbalance_ratio=2.423238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49643, 45708, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=69462 max_load_reduction_pct=0.583190 moved_to_replicas=115752 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[6, 160470, 4442, 3055, 42143, 90268, 60090, 32742] avg=49152.0000 max=160470 max_rank=1 min=6 min_rank=0 imbalance_ratio=3.264771 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49646, 49643, 49644, 49645, 45707] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=110824 max_load_reduction_pct=0.690621 moved_to_replicas=161896 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[37772, 61514, 7, 6629, 65249, 4965, 6, 217074] avg=49152.0000 max=217074 max_rank=7 min=6 min_rank=6 imbalance_ratio=4.416382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[49642, 49644, 49644, 45709, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=167429 max_load_reduction_pct=0.771299 moved_to_replicas=194905 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[15, 66402, 28608, 4004, 189043, 65549, 38393, 1202] avg=49152.0000 max=189043 max_rank=4 min=15 min_rank=0 imbalance_ratio=3.846090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[49645, 49643, 45709, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=139398 max_load_reduction_pct=0.737388 moved_to_replicas=172062 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[9, 294, 82513, 3158, 64226, 6881, 181322, 54813] avg=49152.0000 max=181322 max_rank=6 min=9 min_rank=0 imbalance_ratio=3.689006 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[49644, 49645, 49645, 49645, 49644, 45708, 49642, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=131677 max_load_reduction_pct=0.726205 moved_to_replicas=184298 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[31426, 15962, 32466, 130079, 6919, 69093, 5, 107266] avg=49152.0000 max=130079 max_rank=3 min=5 min_rank=6 imbalance_ratio=2.646464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[46885, 48465, 49644, 49645, 49646, 49644, 49642, 49645] avg=49152.0000 max=49646 max_rank=4 min=46885 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=80433 max_load_reduction_pct=0.618340 moved_to_replicas=157506 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[87059, 6042, 65638, 41076, 138406, 1412, 5, 53578] avg=49152.0000 max=138406 max_rank=4 min=5 min_rank=6 imbalance_ratio=2.815877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[49644, 49642, 49645, 45708, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=88761 max_load_reduction_pct=0.641309 moved_to_replicas=146105 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[63599, 1664, 95263, 80911, 4204, 133068, 14503, 4] avg=49152.0000 max=133068 max_rank=5 min=4 min_rank=7 imbalance_ratio=2.707275 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[49645, 45707, 49644, 49646, 49644, 49642, 49645, 49643] avg=49152.0000 max=49646 max_rank=3 min=45707 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=83422 max_load_reduction_pct=0.626913 moved_to_replicas=174265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[29573, 112306, 810, 13951, 19866, 181, 162612, 53917] avg=49152.0000 max=162612 max_rank=6 min=181 min_rank=5 imbalance_ratio=3.308350 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[45707, 49644, 49646, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45707 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=112966 max_load_reduction_pct=0.694697 moved_to_replicas=179903 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=23 total_expected_recv=1133045 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56448 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=23 total_expected_recv=1127327 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=63744 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=23 total_expected_recv=1131525 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58368 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=23 total_expected_recv=1137881 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=51968 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=23 total_expected_recv=1129377 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59264 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=23 total_expected_recv=1132497 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57984 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=23 total_expected_recv=1120395 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=68864 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=23 total_expected_recv=1131921 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56832 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 18:00:15] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 638.5 | throughput per GPU (TFLOP/s/GPU): 199.8 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.477323E+00 | loss scale: 1.0 | grad norm: 7.143 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=52224 expected_recv=48429 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=52224 expected_recv=47069 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=52224 expected_recv=49498 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=52224 expected_recv=48429 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=52224 expected_recv=47069 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48000 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=52224 expected_recv=49498 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=52224 expected_recv=47631 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=52224 expected_recv=47631 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=52224 expected_recv=47721 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=48000 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=52224 expected_recv=47721 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=46208 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=52224 expected_recv=48890 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=52224 expected_recv=46461 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=52224 expected_recv=48890 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=52224 expected_recv=46461 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=47104 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=52224 expected_recv=49196 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=52224 expected_recv=49196 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=52224 expected_recv=48530 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=52224 expected_recv=47269 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=52224 expected_recv=48530 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=52224 expected_recv=48359 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=52224 expected_recv=47269 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=52224 expected_recv=48359 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=47872 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=52224 expected_recv=46993 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=52224 expected_recv=46993 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=47488 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=52224 expected_recv=47616 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=52224 expected_recv=47738 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=52224 expected_recv=47616 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=52224 expected_recv=47738 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=52224 expected_recv=45818 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=52224 expected_recv=49536 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=52224 expected_recv=45818 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=52224 expected_recv=49536 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=52224 expected_recv=48316 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=52224 expected_recv=48316 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=52224 expected_recv=47035 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=52224 expected_recv=47035 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=48000 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=52224 expected_recv=47985 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=52224 expected_recv=47985 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=52224 expected_recv=47367 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=52224 expected_recv=47367 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=48000 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=52224 expected_recv=46554 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=52224 expected_recv=46554 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=47232 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=52224 expected_recv=48798 expected_overflow=0 exact_recv_per_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=52224 expected_recv=48798 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=52224 expected_recv=47925 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=52224 expected_recv=47428 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=52224 expected_recv=47925 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=52224 expected_recv=47428 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=52224 expected_recv=47343 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=52224 expected_recv=47343 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=47872 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=52224 expected_recv=48377 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=52224 expected_recv=49277 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=52224 expected_recv=48377 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=52224 expected_recv=49277 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=52224 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=52224 expected_recv=49296 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=52224 expected_recv=46055 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=52224 expected_recv=46055 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=46592 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=52224 expected_recv=48640 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=52224 expected_recv=48640 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=49280 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=52224 expected_recv=48442 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=52224 expected_recv=46713 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=52224 expected_recv=48442 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=52224 expected_recv=46713 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=47232 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=52224 expected_recv=46911 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=52224 expected_recv=46911 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=47616 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[53618, 50280, 38742, 49498, 61618, 41387, 44727, 53346] avg=49152.0000 max=61618 max_rank=4 min=38742 min_rank=2 imbalance_ratio=1.253621 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[49644, 49644, 49644, 49498, 49644, 47069, 48429, 49644] avg=49152.0000 max=49644 max_rank=0 min=47069 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11974 max_load_reduction_pct=0.194326 moved_to_replicas=20286 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[76087, 50036, 44491, 58706, 15755, 35365, 50529, 62247] avg=49152.0000 max=76087 max_rank=0 min=15755 min_rank=4 imbalance_ratio=1.547994 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[49644, 49644, 47721, 49643, 49645, 47631, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=47631 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=26442 max_load_reduction_pct=0.347523 moved_to_replicas=49385 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[44271, 57171, 102275, 40202, 32339, 26193, 64034, 26731] avg=49152.0000 max=102275 max_rank=2 min=26193 min_rank=5 imbalance_ratio=2.080790 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[45708, 49643, 49644, 49646, 49644, 49644, 49643, 49644] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=52629 max_load_reduction_pct=0.514583 moved_to_replicas=74548 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[54452, 31307, 19650, 77048, 75265, 29692, 60762, 45040] avg=49152.0000 max=77048 max_rank=3 min=19650 min_rank=2 imbalance_ratio=1.567546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[49645, 48890, 46461, 49644, 49643, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=0 min=46461 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=27403 max_load_reduction_pct=0.355661 moved_to_replicas=68951 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[53237, 44938, 37416, 66123, 55540, 44401, 54439, 37122] avg=49152.0000 max=66123 max_rank=3 min=37122 min_rank=7 imbalance_ratio=1.345276 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[49645, 48530, 47269, 49644, 49644, 49196, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=47269 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16478 max_load_reduction_pct=0.249202 moved_to_replicas=30763 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[66747, 67233, 25268, 34962, 28774, 46814, 38645, 84773] avg=49152.0000 max=84773 max_rank=7 min=25268 min_rank=2 imbalance_ratio=1.724711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[49644, 49643, 49644, 49645, 49644, 46993, 48359, 49644] avg=49152.0000 max=49645 max_rank=3 min=46993 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=35128 max_load_reduction_pct=0.414377 moved_to_replicas=69821 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[55772, 56636, 15999, 58792, 31475, 29174, 55239, 90129] avg=49152.0000 max=90129 max_rank=7 min=15999 min_rank=2 imbalance_ratio=1.833679 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 47616, 47738, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47616 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=40485 max_load_reduction_pct=0.449189 moved_to_replicas=68348 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[57071, 51475, 37591, 19095, 23631, 34106, 43987, 126260] avg=49152.0000 max=126260 max_rank=7 min=19095 min_rank=3 imbalance_ratio=2.568766 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[49643, 49644, 49645, 49643, 49536, 49643, 45818, 49644] avg=49152.0000 max=49645 max_rank=2 min=45818 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=76615 max_load_reduction_pct=0.606803 moved_to_replicas=85874 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[21523, 21530, 64892, 52960, 48316, 36618, 69253, 78124] avg=49152.0000 max=78124 max_rank=7 min=21523 min_rank=0 imbalance_ratio=1.589437 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[49644, 47035, 49644, 49644, 48316, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47035 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=28479 max_load_reduction_pct=0.364536 moved_to_replicas=66653 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[26893, 31514, 101807, 74990, 44561, 31061, 35023, 47367] avg=49152.0000 max=101807 max_rank=2 min=26893 min_rank=0 imbalance_ratio=2.071269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[49644, 49643, 49643, 49645, 47985, 49645, 49644, 47367] avg=49152.0000 max=49645 max_rank=3 min=47367 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=52162 max_load_reduction_pct=0.512362 moved_to_replicas=77509 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[75483, 11376, 18326, 37902, 90301, 55647, 37195, 66986] avg=49152.0000 max=90301 max_rank=4 min=11376 min_rank=1 imbalance_ratio=1.837179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[49643, 46554, 49645, 48798, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=46554 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=40656 max_load_reduction_pct=0.450228 moved_to_replicas=89841 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[22719, 100949, 27548, 54121, 37920, 102454, 37860, 9645] avg=49152.0000 max=102454 max_rank=5 min=9645 min_rank=7 imbalance_ratio=2.084432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=52810 max_load_reduction_pct=0.515451 moved_to_replicas=108592 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[29121, 16382, 97159, 85521, 34742, 50851, 44064, 35376] avg=49152.0000 max=97159 max_rank=2 min=16382 min_rank=1 imbalance_ratio=1.976705 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[49643, 49645, 49643, 49644, 49645, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=47514 max_load_reduction_pct=0.489033 moved_to_replicas=84599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[142196, 31364, 11705, 31441, 17318, 71568, 59060, 28564] avg=49152.0000 max=142196 max_rank=0 min=11705 min_rank=2 imbalance_ratio=2.892985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=92552 max_load_reduction_pct=0.650876 moved_to_replicas=123892 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[28872, 122433, 63648, 15618, 29437, 39624, 40563, 53021] avg=49152.0000 max=122433 max_rank=1 min=15618 min_rank=3 imbalance_ratio=2.490906 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47428, 49643, 47925, 49644] avg=49152.0000 max=49644 max_rank=0 min=47428 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=72789 max_load_reduction_pct=0.594521 moved_to_replicas=90170 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[31264, 54448, 35776, 76195, 99969, 16432, 47343, 31789] avg=49152.0000 max=99969 max_rank=4 min=16432 min_rank=5 imbalance_ratio=2.033875 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[48377, 49644, 49277, 49644, 49644, 49644, 47343, 49643] avg=49152.0000 max=49644 max_rank=1 min=47343 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=50325 max_load_reduction_pct=0.503406 moved_to_replicas=81680 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[4439, 43292, 82806, 81562, 62151, 71086, 27375, 20505] avg=49152.0000 max=82806 max_rank=2 min=4439 min_rank=0 imbalance_ratio=1.684692 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=33161 max_load_reduction_pct=0.400466 moved_to_replicas=99029 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[41762, 16414, 24328, 62003, 57826, 72712, 30930, 87241] avg=49152.0000 max=87241 max_rank=7 min=16414 min_rank=1 imbalance_ratio=1.774923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 45708, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=37597 max_load_reduction_pct=0.430956 moved_to_replicas=81206 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[26905, 43695, 77476, 14261, 52004, 116475, 26668, 35732] avg=49152.0000 max=116475 max_rank=5 min=14261 min_rank=3 imbalance_ratio=2.369690 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[49644, 46055, 49644, 49644, 49644, 49644, 49296, 49645] avg=49152.0000 max=49645 max_rank=7 min=46055 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=66830 max_load_reduction_pct=0.573771 moved_to_replicas=97023 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[53906, 84502, 74665, 50249, 29911, 9532, 59587, 30864] avg=49152.0000 max=84502 max_rank=1 min=9532 min_rank=5 imbalance_ratio=1.719198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49645, 45708, 49643, 49644] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=34857 max_load_reduction_pct=0.412499 moved_to_replicas=74689 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[30981, 62875, 39034, 29360, 41479, 100008, 68373, 21106] avg=49152.0000 max=100008 max_rank=5 min=21106 min_rank=7 imbalance_ratio=2.034668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 45708, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=50363 max_load_reduction_pct=0.503590 moved_to_replicas=82324 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[18754, 30522, 65640, 64078, 49838, 26273, 67201, 70910] avg=49152.0000 max=70910 max_rank=7 min=18754 min_rank=0 imbalance_ratio=1.442668 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[49645, 46713, 49643, 49643, 49644, 48640, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=46713 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=21265 max_load_reduction_pct=0.299887 moved_to_replicas=69447 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[101702, 57508, 24481, 20016, 69062, 57195, 31774, 31478] avg=49152.0000 max=101702 max_rank=0 min=20016 min_rank=3 imbalance_ratio=2.069132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[49644, 49644, 46911, 49644, 49643, 49644, 48442, 49644] avg=49152.0000 max=49644 max_rank=0 min=46911 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=52058 max_load_reduction_pct=0.511868 moved_to_replicas=86891 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=23 total_expected_recv=1122666 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=64512 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=23 total_expected_recv=1122043 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=66304 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=23 total_expected_recv=1123786 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=63360 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=23 total_expected_recv=1127280 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=60416 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=23 total_expected_recv=1136609 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=50816 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=23 total_expected_recv=1139534 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=48896 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=23 total_expected_recv=1131230 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56192 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=23 total_expected_recv=1140820 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=47616 peak_demand=49646 required_capacity=1.010050 + [2026-08-26 18:00:16] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 634.1 | throughput per GPU (TFLOP/s/GPU): 201.2 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.119604E+01 | loss scale: 1.0 | grad norm: 396.977 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=52224 expected_recv=47706 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=52224 expected_recv=47646 expected_overflow=0 exact_recv_per_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=52224 expected_recv=47706 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=52224 expected_recv=47646 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=52224 expected_recv=46853 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=52224 expected_recv=48499 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=52224 expected_recv=46853 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=52224 expected_recv=48499 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=47616 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=52224 expected_recv=47476 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=52224 expected_recv=47476 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=52224 expected_recv=49326 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=52224 expected_recv=48192 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=52224 expected_recv=49326 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=52224 expected_recv=48192 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=52224 expected_recv=47943 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=52224 expected_recv=47943 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=52224 expected_recv=49341 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=52224 expected_recv=49341 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=52224 expected_recv=47713 expected_overflow=0 exact_recv_per_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=52224 expected_recv=47713 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=52224 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=52224 expected_recv=49200 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=52224 expected_recv=46151 expected_overflow=0 exact_recv_per_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=52224 expected_recv=46151 actual_valid_recv=46720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=46720 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=52224 expected_recv=49589 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=52224 expected_recv=49605 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=52224 expected_recv=45763 expected_overflow=0 exact_recv_per_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=52224 expected_recv=49589 actual_valid_recv=50688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=52224 expected_recv=49605 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=52224 expected_recv=45763 actual_valid_recv=46720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50688 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=46720 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=52224 expected_recv=45746 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=52224 expected_recv=45746 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=46208 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=52224 expected_recv=47277 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=52224 expected_recv=47277 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=52224 expected_recv=48075 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=52224 expected_recv=48075 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=52224 expected_recv=49475 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=52224 expected_recv=49475 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=52224 expected_recv=45876 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=52224 expected_recv=45876 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=46592 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=46208 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50688 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50688 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=52224 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=52224 expected_recv=45709 actual_valid_recv=46720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=52224 expected_recv=49546 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=46720 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=52224 expected_recv=49546 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=52224 expected_recv=45807 expected_overflow=0 exact_recv_per_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=52224 expected_recv=46081 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=52224 expected_recv=45807 actual_valid_recv=46720 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=52224 expected_recv=46081 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=46720 slack_slots=5504 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=46592 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=52224 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=52224 expected_recv=49270 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=52224 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=52224 expected_recv=45709 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=52224 expected_recv=47662 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=46592 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=52224 expected_recv=47662 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=52224 expected_recv=47690 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=52224 expected_recv=47690 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=52224 expected_recv=47387 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=52224 expected_recv=47387 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=52224 expected_recv=47967 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=52224 expected_recv=47967 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=46592 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=52224 expected_recv=48935 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=52224 expected_recv=48935 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=52224 expected_recv=46414 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=52224 expected_recv=46414 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=47104 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=52224 expected_recv=48233 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=52224 expected_recv=48233 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=52224 expected_recv=47123 expected_overflow=0 exact_recv_per_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=52224 expected_recv=47123 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=52224 expected_recv=49641 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=47616 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=52224 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=52224 expected_recv=49641 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=52224 expected_recv=49177 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=52224 expected_recv=46176 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=52224 expected_recv=46176 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[47646, 68526, 33994, 43831, 55839, 32492, 47297, 63591] avg=49152.0000 max=68526 max_rank=1 min=32492 min_rank=5 imbalance_ratio=1.394165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[47646, 49644, 49644, 49645, 49643, 49644, 47706, 49644] avg=49152.0000 max=49645 max_rank=3 min=47646 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=18881 max_load_reduction_pct=0.275530 moved_to_replicas=39024 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[63897, 54618, 36692, 51654, 9781, 23972, 98136, 54466] avg=49152.0000 max=98136 max_rank=6 min=9781 min_rank=4 imbalance_ratio=1.996582 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[49644, 49643, 48499, 49644, 49645, 46853, 49644, 49644] avg=49152.0000 max=49645 max_rank=4 min=46853 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=48491 max_load_reduction_pct=0.494120 moved_to_replicas=74551 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[21927, 34267, 135134, 56529, 44767, 33814, 14425, 52353] avg=49152.0000 max=135134 max_rank=2 min=14425 min_rank=6 imbalance_ratio=2.749308 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[49645, 48192, 49643, 49645, 47476, 49326, 49645, 49644] avg=49152.0000 max=49645 max_rank=0 min=47476 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=85489 max_load_reduction_pct=0.632624 moved_to_replicas=95084 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[34062, 63294, 60596, 39801, 18980, 77498, 15618, 83367] avg=49152.0000 max=83367 max_rank=7 min=15618 min_rank=6 imbalance_ratio=1.696106 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[47713, 49643, 49644, 49644, 47943, 49644, 49341, 49644] avg=49152.0000 max=49644 max_rank=2 min=47713 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33723 max_load_reduction_pct=0.404513 moved_to_replicas=86179 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[34761, 1272, 101319, 19654, 58620, 73985, 102187, 1418] avg=49152.0000 max=102187 max_rank=6 min=1272 min_rank=1 imbalance_ratio=2.079000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[45708, 49644, 49643, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=52542 max_load_reduction_pct=0.514175 moved_to_replicas=137535 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[158541, 30616, 72282, 20101, 13454, 56980, 21576, 19666] avg=49152.0000 max=158541 max_rank=0 min=13454 min_rank=4 imbalance_ratio=3.225525 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[49644, 46151, 49645, 49644, 49644, 49644, 49644, 49200] avg=49152.0000 max=49645 max_rank=2 min=46151 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=108896 max_load_reduction_pct=0.686863 moved_to_replicas=138871 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[8164, 164691, 9915, 1729, 42480, 68959, 32368, 64910] avg=49152.0000 max=164691 max_rank=1 min=1729 min_rank=3 imbalance_ratio=3.350647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[49645, 49644, 49645, 49643, 45708, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=115046 max_load_reduction_pct=0.698557 moved_to_replicas=149628 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[3998, 32802, 101868, 62605, 17732, 21, 114879, 59311] avg=49152.0000 max=114879 max_rank=6 min=21 min_rank=5 imbalance_ratio=2.337219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[49589, 45763, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=45763 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=65235 max_load_reduction_pct=0.567858 moved_to_replicas=140087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[85338, 67624, 56658, 22729, 27017, 53518, 3038, 77294] avg=49152.0000 max=85338 max_rank=0 min=3038 min_rank=6 imbalance_ratio=1.736206 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[49644, 49645, 49644, 49643, 49605, 49644, 45746, 49645] avg=49152.0000 max=49645 max_rank=1 min=45746 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=35693 max_load_reduction_pct=0.418254 moved_to_replicas=92212 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[56502, 3566, 133405, 23158, 96424, 4577, 74561, 1023] avg=49152.0000 max=133405 max_rank=2 min=1023 min_rank=7 imbalance_ratio=2.714132 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[49644, 49643, 49644, 48075, 49644, 47277, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47277 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=83760 max_load_reduction_pct=0.627863 moved_to_replicas=162316 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[8928, 54788, 44331, 2794, 196619, 20930, 64785, 41] avg=49152.0000 max=196619 max_rank=4 min=41 min_rank=7 imbalance_ratio=4.000224 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[45876, 49644, 49475, 49645, 49646, 49644, 49644, 49642] avg=49152.0000 max=49646 max_rank=4 min=45876 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=146973 max_load_reduction_pct=0.747502 moved_to_replicas=167260 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[525, 30, 65525, 114091, 10067, 65528, 44106, 93344] avg=49152.0000 max=114091 max_rank=3 min=30 min_rank=1 imbalance_ratio=2.321187 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[49644, 49643, 49645, 49644, 49645, 49644, 45708, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=64446 max_load_reduction_pct=0.564865 moved_to_replicas=139912 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[14276, 131129, 2082, 67354, 669, 59253, 55358, 63095] avg=49152.0000 max=131129 max_rank=1 min=669 min_rank=4 imbalance_ratio=2.667826 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[45709, 49646, 49642, 49644, 49643, 49645, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=45709 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=81483 max_load_reduction_pct=0.621396 moved_to_replicas=127969 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[68514, 40810, 123355, 19722, 4833, 125557, 5488, 4937] avg=49152.0000 max=125557 max_rank=5 min=4833 min_rank=4 imbalance_ratio=2.554464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[49642, 45807, 49645, 49646, 49642, 49645, 49643, 49546] avg=49152.0000 max=49646 max_rank=3 min=45807 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=75911 max_load_reduction_pct=0.604594 moved_to_replicas=168494 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[1279, 29622, 64723, 127734, 51, 118979, 46081, 4747] avg=49152.0000 max=127734 max_rank=3 min=51 min_rank=4 imbalance_ratio=2.598755 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[49644, 49270, 49645, 49645, 49645, 49644, 46081, 49642] avg=49152.0000 max=49645 max_rank=2 min=46081 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=78089 max_load_reduction_pct=0.611341 moved_to_replicas=162504 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[17, 13486, 18942, 68649, 5825, 155084, 66316, 64897] avg=49152.0000 max=155084 max_rank=5 min=17 min_rank=0 imbalance_ratio=3.155192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[49644, 45709, 49644, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=3 min=45709 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=105439 max_load_reduction_pct=0.679883 moved_to_replicas=156370 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[14, 24, 62361, 53, 52560, 131021, 31168, 116015] avg=49152.0000 max=131021 max_rank=5 min=14 min_rank=0 imbalance_ratio=2.665629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[49643, 49645, 49644, 47690, 49644, 49644, 47662, 49644] avg=49152.0000 max=49645 max_rank=1 min=47662 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=81376 max_load_reduction_pct=0.621091 moved_to_replicas=163381 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[131057, 60558, 165532, 2017, 23929, 6541, 26, 3556] avg=49152.0000 max=165532 max_rank=2 min=26 min_rank=6 imbalance_ratio=3.367757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[49645, 49643, 49645, 49643, 49644, 49645, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115887 max_load_reduction_pct=0.700088 moved_to_replicas=208215 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[25, 32067, 65518, 129335, 65543, 34930, 53, 65745] avg=49152.0000 max=129335 max_rank=3 min=25 min_rank=0 imbalance_ratio=2.631327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[49644, 47967, 49645, 49644, 49643, 49643, 47387, 49643] avg=49152.0000 max=49645 max_rank=2 min=47387 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=79690 max_load_reduction_pct=0.616152 moved_to_replicas=127565 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[13, 128230, 96580, 56738, 103451, 8140, 37, 27] avg=49152.0000 max=128230 max_rank=1 min=13 min_rank=0 imbalance_ratio=2.608846 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49644, 49644, 45708] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=78586 max_load_reduction_pct=0.612852 moved_to_replicas=186423 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[6085, 10458, 32918, 30019, 193431, 65662, 26509, 28134] avg=49152.0000 max=193431 max_rank=4 min=6085 min_rank=0 imbalance_ratio=3.935364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[49645, 46414, 48935, 49644, 49645, 49645, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46414 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=143786 max_load_reduction_pct=0.743345 moved_to_replicas=159805 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[16, 65527, 65510, 13, 65518, 66963, 63560, 66109] avg=49152.0000 max=66963 max_rank=5 min=13 min_rank=3 imbalance_ratio=1.362366 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[48233, 49643, 49643, 47123, 49643, 49644, 49644, 49643] avg=49152.0000 max=49644 max_rank=5 min=47123 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=17319 max_load_reduction_pct=0.258635 moved_to_replicas=95323 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[135, 45546, 65728, 63807, 65350, 4993, 84522, 63135] avg=49152.0000 max=84522 max_rank=6 min=135 min_rank=0 imbalance_ratio=1.719604 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[49177, 46176, 49645, 49644, 49645, 49641, 49643, 49645] avg=49152.0000 max=49645 max_rank=2 min=46176 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=34877 max_load_reduction_pct=0.412638 moved_to_replicas=94322 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=23 total_expected_recv=1133970 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=52224 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=23 total_expected_recv=1123931 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=63104 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=23 total_expected_recv=1133392 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=53888 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=23 total_expected_recv=1139792 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=48384 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=23 total_expected_recv=1124312 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=62464 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=23 total_expected_recv=1116463 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=69504 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=23 total_expected_recv=1136336 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=50432 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=23 total_expected_recv=1135772 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=52608 peak_demand=49646 required_capacity=1.010050 + [2026-08-26 18:00:16] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 640.7 | throughput per GPU (TFLOP/s/GPU): 199.1 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.367741E+00 | loss scale: 1.0 | grad norm: 5.090 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=52224 expected_recv=47434 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=52224 expected_recv=48655 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=52224 expected_recv=47434 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=47872 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=52224 expected_recv=48907 expected_overflow=0 exact_recv_per_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=52224 expected_recv=48655 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=52224 expected_recv=48907 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=52224 expected_recv=49426 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=52224 expected_recv=49426 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=52224 expected_recv=45949 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=52224 expected_recv=45949 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=52224 expected_recv=49621 expected_overflow=0 exact_recv_per_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=52224 expected_recv=49621 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=52224 expected_recv=46856 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=52224 expected_recv=46856 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=47232 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=52224 expected_recv=48496 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=52224 expected_recv=48496 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=52224 expected_recv=47698 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=52224 expected_recv=47698 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=52224 expected_recv=47654 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=52224 expected_recv=47654 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=52224 expected_recv=48202 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=52224 expected_recv=48202 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=52224 expected_recv=47149 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=52224 expected_recv=47149 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=47872 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=52224 expected_recv=46566 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=52224 expected_recv=47601 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=52224 expected_recv=46566 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=52224 expected_recv=47601 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=52224 expected_recv=48785 expected_overflow=0 exact_recv_per_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=47104 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=52224 expected_recv=48785 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=52224 expected_recv=47863 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=52224 expected_recv=47753 expected_overflow=0 exact_recv_per_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=52224 expected_recv=47863 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=52224 expected_recv=47753 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=52224 expected_recv=47491 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=52224 expected_recv=47491 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=52224 expected_recv=49387 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=52224 expected_recv=49387 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=52224 expected_recv=45963 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=52224 expected_recv=45963 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=46848 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=52224 expected_recv=46448 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=52224 expected_recv=48902 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=52224 expected_recv=46448 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=52224 expected_recv=48902 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=47232 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=52224 expected_recv=46856 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=52224 expected_recv=46856 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=52224 expected_recv=48940 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=52224 expected_recv=48496 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=47744 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=52224 expected_recv=48940 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=52224 expected_recv=48496 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=52224 expected_recv=46410 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=52224 expected_recv=46410 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=47104 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=52224 expected_recv=48994 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=52224 expected_recv=46357 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=52224 expected_recv=48687 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=52224 expected_recv=48994 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=52224 expected_recv=46357 actual_valid_recv=46976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=52224 expected_recv=48687 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=46976 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=49280 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=52224 expected_recv=46664 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=52224 expected_recv=46664 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=47232 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=46080 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=52224 expected_recv=46977 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=52224 expected_recv=46977 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=47360 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=52224 expected_recv=48371 expected_overflow=0 exact_recv_per_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=52224 expected_recv=48371 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=46208 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=52224 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=52224 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=52224 expected_recv=45706 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=52224 expected_recv=45706 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=52224 expected_recv=46484 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=52224 expected_recv=46484 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=47104 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=52224 expected_recv=48866 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=52224 expected_recv=48375 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=52224 expected_recv=48866 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=52224 expected_recv=48375 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=52224 expected_recv=46980 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=52224 expected_recv=46980 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=47616 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[48907, 65943, 36862, 39707, 55149, 32535, 46967, 67146] avg=49152.0000 max=67146 max_rank=7 min=32535 min_rank=5 imbalance_ratio=1.366089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[48907, 49643, 49645, 49643, 49645, 48655, 47434, 49644] avg=49152.0000 max=49645 max_rank=2 min=47434 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=17501 max_load_reduction_pct=0.260641 moved_to_replicas=39306 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[49621, 64862, 49426, 41524, 14833, 31275, 100004, 41671] avg=49152.0000 max=100004 max_rank=6 min=14833 min_rank=4 imbalance_ratio=2.034587 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[49621, 49644, 49426, 45949, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45949 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=50360 max_load_reduction_pct=0.503580 moved_to_replicas=65578 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[13374, 33963, 82913, 77702, 75528, 24250, 7551, 77935] avg=49152.0000 max=82913 max_rank=2 min=7551 min_rank=6 imbalance_ratio=1.686869 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=33268 max_load_reduction_pct=0.401240 moved_to_replicas=115502 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[24626, 100620, 39708, 48496, 3716, 73008, 32097, 70945] avg=49152.0000 max=100620 max_rank=1 min=3716 min_rank=4 imbalance_ratio=2.047119 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[49645, 49644, 46856, 48496, 49644, 49643, 49643, 49645] avg=49152.0000 max=49645 max_rank=0 min=46856 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=50975 max_load_reduction_pct=0.506609 moved_to_replicas=95641 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[64215, 12174, 50404, 58367, 23644, 51560, 123992, 8860] avg=49152.0000 max=123992 max_rank=6 min=8860 min_rank=7 imbalance_ratio=2.522624 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[49643, 47654, 49644, 49645, 47698, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=3 min=47654 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=74347 max_load_reduction_pct=0.599611 moved_to_replicas=100318 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[58187, 20825, 151854, 40930, 37512, 12370, 25304, 46234] avg=49152.0000 max=151854 max_rank=2 min=12370 min_rank=5 imbalance_ratio=3.089478 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 47149, 49645, 49644, 49644, 48202] avg=49152.0000 max=49645 max_rank=4 min=47149 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=102209 max_load_reduction_pct=0.673074 moved_to_replicas=110753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[28463, 71822, 3529, 816, 64423, 45896, 108300, 69967] avg=49152.0000 max=108300 max_rank=6 min=816 min_rank=3 imbalance_ratio=2.203369 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[48785, 49643, 49644, 49644, 49645, 46566, 49644, 49645] avg=49152.0000 max=49645 max_rank=4 min=46566 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=58655 max_load_reduction_pct=0.541597 moved_to_replicas=115936 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[9958, 125629, 14527, 65411, 47601, 820, 108683, 20587] avg=49152.0000 max=125629 max_rank=1 min=820 min_rank=5 imbalance_ratio=2.555929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[47753, 49642, 49644, 49644, 47601, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=2 min=47601 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=75985 max_load_reduction_pct=0.604836 moved_to_replicas=150791 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[101789, 71807, 52081, 8772, 68663, 6260, 26406, 57438] avg=49152.0000 max=101789 max_rank=0 min=6260 min_rank=5 imbalance_ratio=2.070903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[49644, 49644, 49644, 47491, 49643, 49644, 47863, 49643] avg=49152.0000 max=49644 max_rank=0 min=47491 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=52145 max_load_reduction_pct=0.512285 moved_to_replicas=103558 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[67554, 9338, 126446, 2535, 63798, 19851, 61015, 42679] avg=49152.0000 max=126446 max_rank=2 min=2535 min_rank=3 imbalance_ratio=2.572550 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[49644, 49387, 49645, 49644, 49645, 45963, 49644, 49644] avg=49152.0000 max=49645 max_rank=2 min=45963 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76801 max_load_reduction_pct=0.607382 moved_to_replicas=120237 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[4163, 10919, 16398, 172398, 155742, 14291, 529, 18776] avg=49152.0000 max=172398 max_rank=3 min=529 min_rank=6 imbalance_ratio=3.507446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[49645, 46448, 49644, 49644, 49644, 49645, 49644, 48902] avg=49152.0000 max=49645 max_rank=0 min=46448 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=122753 max_load_reduction_pct=0.712033 moved_to_replicas=228852 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[72376, 544, 71454, 66343, 11194, 48496, 114953, 7856] avg=49152.0000 max=114953 max_rank=6 min=544 min_rank=1 imbalance_ratio=2.338725 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49643, 48496, 49644, 46856] avg=49152.0000 max=49645 max_rank=3 min=46856 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=65308 max_load_reduction_pct=0.568128 moved_to_replicas=126550 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[11750, 131325, 104228, 74126, 6289, 5355, 14, 60129] avg=49152.0000 max=131325 max_rank=1 min=14 min_rank=6 imbalance_ratio=2.671814 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 48940, 46410, 49644, 49645] avg=49152.0000 max=49645 max_rank=3 min=46410 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81680 max_load_reduction_pct=0.621968 moved_to_replicas=171232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[7005, 13143, 117045, 57628, 62356, 65718, 62981, 7340] avg=49152.0000 max=117045 max_rank=2 min=7005 min_rank=0 imbalance_ratio=2.381287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[49643, 48994, 49644, 49644, 49645, 49644, 49645, 46357] avg=49152.0000 max=49645 max_rank=4 min=46357 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=67400 max_load_reduction_pct=0.575847 moved_to_replicas=117508 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[1355, 62199, 72077, 5391, 12040, 116803, 54951, 68400] avg=49152.0000 max=116803 max_rank=5 min=1355 min_rank=0 imbalance_ratio=2.376363 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[49646, 49644, 49643, 46664, 48687, 49644, 49644, 49644] avg=49152.0000 max=49646 max_rank=0 min=46664 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=67157 max_load_reduction_pct=0.574960 moved_to_replicas=126210 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[2708, 2155, 89476, 104118, 110, 168080, 26038, 531] avg=49152.0000 max=168080 max_rank=5 min=110 min_rank=4 imbalance_ratio=3.419596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49645, 49643, 49644, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=118435 max_load_reduction_pct=0.704635 moved_to_replicas=212742 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[994, 68466, 52147, 67300, 65506, 29323, 73, 109407] avg=49152.0000 max=109407 max_rank=7 min=73 min_rank=6 imbalance_ratio=2.225891 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[48371, 49645, 49644, 49646, 49644, 46977, 49645, 49644] avg=49152.0000 max=49646 max_rank=3 min=46977 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=59761 max_load_reduction_pct=0.546226 moved_to_replicas=114606 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[64841, 4, 77987, 11385, 5113, 119462, 29, 114395] avg=49152.0000 max=119462 max_rank=5 min=4 min_rank=1 imbalance_ratio=2.430461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49644, 49644, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=69817 max_load_reduction_pct=0.584429 moved_to_replicas=178109 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[33, 13898, 105507, 64423, 92864, 61438, 2523, 52530] avg=49152.0000 max=105507 max_rank=2 min=33 min_rank=0 imbalance_ratio=2.146545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[49645, 45708, 49643, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=55862 max_load_reduction_pct=0.529463 moved_to_replicas=128542 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[71734, 126344, 2121, 94610, 66463, 28531, 71, 3342] avg=49152.0000 max=126344 max_rank=1 min=71 min_rank=6 imbalance_ratio=2.570475 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[49644, 49643, 49644, 49643, 49645, 45707, 49644, 49646] avg=49152.0000 max=49646 max_rank=7 min=45707 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=76698 max_load_reduction_pct=0.607057 moved_to_replicas=160575 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[65532, 19563, 5036, 19, 120056, 66163, 65381, 51466] avg=49152.0000 max=120056 max_rank=4 min=19 min_rank=3 imbalance_ratio=2.442546 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[49646, 45706, 49645, 49644, 49644, 49642, 49645, 49644] avg=49152.0000 max=49646 max_rank=0 min=45706 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=70410 max_load_reduction_pct=0.586476 moved_to_replicas=120378 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[54260, 65526, 2984, 602, 65938, 131005, 55799, 17102] avg=49152.0000 max=131005 max_rank=5 min=602 min_rank=3 imbalance_ratio=2.665304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[49644, 49643, 46484, 49644, 49645, 49646, 49644, 48866] avg=49152.0000 max=49646 max_rank=5 min=46484 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=81359 max_load_reduction_pct=0.621037 moved_to_replicas=124308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[109, 130821, 65529, 72015, 65516, 891, 665, 57670] avg=49152.0000 max=130821 max_rank=1 min=109 min_rank=0 imbalance_ratio=2.661560 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[49645, 49643, 49642, 49645, 49643, 46980, 48375, 49643] avg=49152.0000 max=49645 max_rank=0 min=46980 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=81176 max_load_reduction_pct=0.620512 moved_to_replicas=143331 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=23 total_expected_recv=1136167 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=52224 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=23 total_expected_recv=1136554 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=50048 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=23 total_expected_recv=1127840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59776 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=23 total_expected_recv=1135644 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=51456 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=23 total_expected_recv=1120414 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=65920 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=23 total_expected_recv=1133099 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55808 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=23 total_expected_recv=1128842 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58496 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=23 total_expected_recv=1125408 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=62848 peak_demand=49646 required_capacity=1.010050 + [2026-08-26 18:00:17] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 642.9 | throughput per GPU (TFLOP/s/GPU): 198.5 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.973585E+00 | loss scale: 1.0 | grad norm: 3.690 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=52224 expected_recv=48198 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=52224 expected_recv=47787 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=52224 expected_recv=48198 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=52224 expected_recv=49010 expected_overflow=0 exact_recv_per_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=52224 expected_recv=47787 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=52224 expected_recv=49010 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=52224 expected_recv=48741 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=52224 expected_recv=48901 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=52224 expected_recv=48741 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=49280 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=52224 expected_recv=48901 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=52224 expected_recv=47354 expected_overflow=0 exact_recv_per_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=52224 expected_recv=47354 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=47872 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=52224 expected_recv=47341 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=52224 expected_recv=48768 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=52224 expected_recv=47341 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=52224 expected_recv=48887 expected_overflow=0 exact_recv_per_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=52224 expected_recv=48768 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=48000 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=52224 expected_recv=48887 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=52224 expected_recv=47922 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=52224 expected_recv=47922 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=52224 expected_recv=47430 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=52224 expected_recv=47430 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=52224 expected_recv=45710 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=52224 expected_recv=45710 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=46848 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=52224 expected_recv=48973 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=52224 expected_recv=48456 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=52224 expected_recv=48973 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=52224 expected_recv=48456 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=52224 expected_recv=47567 expected_overflow=0 exact_recv_per_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=52224 expected_recv=47567 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=52224 expected_recv=47079 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=52224 expected_recv=48276 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=52224 expected_recv=47079 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=52224 expected_recv=48276 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=47616 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=46208 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=52224 expected_recv=49647 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=52224 expected_recv=49647 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=46592 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=52224 expected_recv=45711 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=52224 expected_recv=49641 expected_overflow=0 exact_recv_per_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=52224 expected_recv=45711 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=52224 expected_recv=49641 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=46592 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=52224 expected_recv=47071 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=52224 expected_recv=47071 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=47616 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=52224 expected_recv=48155 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=52224 expected_recv=48281 expected_overflow=0 exact_recv_per_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=52224 expected_recv=48155 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=52224 expected_recv=48281 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=52224 expected_recv=47199 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=52224 expected_recv=47199 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=47872 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=46080 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=52224 expected_recv=45710 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=52224 expected_recv=45710 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=46208 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=52224 expected_recv=48158 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=52224 expected_recv=48158 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=52224 expected_recv=47194 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=52224 expected_recv=48199 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=52224 expected_recv=47194 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=52224 expected_recv=48199 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=47872 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=52224 expected_recv=47154 expected_overflow=0 exact_recv_per_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=52224 expected_recv=47154 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=47488 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=52224 expected_recv=49053 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=52224 expected_recv=49053 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=52224 expected_recv=46298 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=52224 expected_recv=46298 actual_valid_recv=46976 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=52224 expected_recv=45762 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=46976 slack_slots=5248 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=52224 expected_recv=45762 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=52224 expected_recv=49588 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=52224 expected_recv=49588 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=52224 expected_recv=48698 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=52224 expected_recv=48698 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=46080 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=52224 expected_recv=46908 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=52224 expected_recv=46908 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=52224 expected_recv=49392 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=47232 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=52224 expected_recv=49392 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=52224 expected_recv=49330 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=52224 expected_recv=49330 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=52224 expected_recv=48442 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=52224 expected_recv=47223 expected_overflow=0 exact_recv_per_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=52224 expected_recv=48442 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=52224 expected_recv=47223 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[49010, 53723, 42429, 40396, 52258, 40272, 51335, 63793] avg=49152.0000 max=63793 max_rank=7 min=40272 min_rank=5 imbalance_ratio=1.297872 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[49010, 49645, 48198, 47787, 49644, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47787 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14148 max_load_reduction_pct=0.221780 moved_to_replicas=22533 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[46851, 61199, 63649, 31841, 34736, 48901, 63673, 42366] avg=49152.0000 max=63673 max_rank=6 min=31841 min_rank=3 imbalance_ratio=1.295431 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[47354, 49644, 49644, 49645, 48741, 48901, 49644, 49643] avg=49152.0000 max=49645 max_rank=3 min=47354 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=14028 max_load_reduction_pct=0.220313 moved_to_replicas=39589 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[31962, 45835, 51150, 48768, 91061, 38405, 16787, 69248] avg=49152.0000 max=91061 max_rank=4 min=16787 min_rank=6 imbalance_ratio=1.852641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[48887, 47341, 49644, 48768, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47341 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=41416 max_load_reduction_pct=0.454816 moved_to_replicas=62527 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[33085, 97275, 52715, 50836, 15315, 71969, 28854, 43167] avg=49152.0000 max=97275 max_rank=1 min=15315 min_rank=4 imbalance_ratio=1.979065 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49643, 47922, 47430] avg=49152.0000 max=49645 max_rank=0 min=47430 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=47630 max_load_reduction_pct=0.489643 moved_to_replicas=74219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[64863, 32215, 28689, 72185, 15753, 51315, 98777, 29419] avg=49152.0000 max=98777 max_rank=6 min=15753 min_rank=4 imbalance_ratio=2.009623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[49643, 49643, 45710, 49644, 49643, 49644, 49643, 49646] avg=49152.0000 max=49646 max_rank=7 min=45710 min_rank=2 imbalance_ratio=1.010050 max_load_reduction=49131 max_load_reduction_pct=0.497393 moved_to_replicas=88564 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[47567, 31411, 75592, 66724, 42518, 40741, 41492, 47171] avg=49152.0000 max=75592 max_rank=2 min=31411 min_rank=1 imbalance_ratio=1.537923 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[47567, 49644, 49644, 49645, 49644, 48456, 49643, 48973] avg=49152.0000 max=49645 max_rank=3 min=47567 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=25947 max_load_reduction_pct=0.343251 moved_to_replicas=43028 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[51994, 18049, 12810, 4062, 57777, 97392, 112187, 38945] avg=49152.0000 max=112187 max_rank=6 min=4062 min_rank=3 imbalance_ratio=2.282450 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[49644, 48276, 49643, 49645, 49643, 49643, 49643, 47079] avg=49152.0000 max=49645 max_rank=3 min=47079 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62542 max_load_reduction_pct=0.557480 moved_to_replicas=120774 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[69928, 44593, 27226, 100225, 18034, 35513, 77236, 20461] avg=49152.0000 max=100225 max_rank=3 min=18034 min_rank=4 imbalance_ratio=2.039083 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[49645, 45708, 49644, 49644, 49644, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=50580 max_load_reduction_pct=0.504665 moved_to_replicas=98457 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[162484, 41518, 32465, 5876, 77648, 1303, 18291, 53631] avg=49152.0000 max=162484 max_rank=0 min=1303 min_rank=5 imbalance_ratio=3.305745 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[49641, 45708, 49644, 49643, 49644, 49645, 49647, 49644] avg=49152.0000 max=49647 max_rank=6 min=45708 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=112837 max_load_reduction_pct=0.694450 moved_to_replicas=144831 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[113954, 36329, 17005, 5336, 125960, 14375, 9974, 70283] avg=49152.0000 max=125960 max_rank=4 min=5336 min_rank=3 imbalance_ratio=2.562663 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[49644, 49644, 49643, 49644, 49644, 49643, 45711, 49643] avg=49152.0000 max=49644 max_rank=0 min=45711 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=76316 max_load_reduction_pct=0.605875 moved_to_replicas=161265 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[2714, 21493, 672, 99488, 75222, 76664, 2564, 114399] avg=49152.0000 max=114399 max_rank=7 min=672 min_rank=2 imbalance_ratio=2.327454 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[48281, 47071, 49644, 49645, 49644, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=3 min=47071 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=64754 max_load_reduction_pct=0.566036 moved_to_replicas=167197 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[68054, 657, 100299, 62309, 17242, 28788, 107428, 8439] avg=49152.0000 max=107428 max_rank=6 min=657 min_rank=1 imbalance_ratio=2.185628 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[49643, 49644, 49644, 49643, 48155, 47199, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=47199 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=57784 max_load_reduction_pct=0.537886 moved_to_replicas=139514 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[42114, 50643, 60306, 154315, 29751, 38910, 16047, 1130] avg=49152.0000 max=154315 max_rank=3 min=1130 min_rank=7 imbalance_ratio=3.139547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[45708, 49644, 49644, 49643, 49643, 49644, 49645, 49645] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=104670 max_load_reduction_pct=0.678288 moved_to_replicas=116332 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[10746, 102490, 19364, 8636, 60981, 15707, 83147, 92145] avg=49152.0000 max=102490 max_rank=1 min=8636 min_rank=3 imbalance_ratio=2.085164 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[49645, 49643, 45710, 49644, 49643, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45710 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=52845 max_load_reduction_pct=0.515611 moved_to_replicas=140187 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[56501, 32275, 10957, 2209, 23038, 159796, 60282, 48158] avg=49152.0000 max=159796 max_rank=5 min=2209 min_rank=3 imbalance_ratio=3.251058 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[49644, 49645, 47194, 49644, 49644, 49644, 49643, 48158] avg=49152.0000 max=49645 max_rank=1 min=47194 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=110151 max_load_reduction_pct=0.689323 moved_to_replicas=127647 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[47154, 5626, 7913, 126420, 3979, 169883, 31188, 1053] avg=49152.0000 max=169883 max_rank=5 min=1053 min_rank=7 imbalance_ratio=3.456278 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[47154, 49645, 49644, 49643, 49643, 49644, 48199, 49644] avg=49152.0000 max=49645 max_rank=1 min=47154 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=120238 max_load_reduction_pct=0.707769 moved_to_replicas=197015 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[25619, 119950, 8120, 129384, 65688, 1791, 43, 42621] avg=49152.0000 max=129384 max_rank=3 min=43 min_rank=6 imbalance_ratio=2.632324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[49643, 49645, 49645, 49644, 49644, 49642, 49645, 45708] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=79739 max_load_reduction_pct=0.616297 moved_to_replicas=166090 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[30, 5990, 144154, 68965, 127349, 3479, 29613, 13636] avg=49152.0000 max=144154 max_rank=2 min=30 min_rank=0 imbalance_ratio=2.932821 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[49642, 49644, 49644, 49644, 49646, 46298, 49053, 49645] avg=49152.0000 max=49646 max_rank=4 min=46298 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=94508 max_load_reduction_pct=0.655604 moved_to_replicas=191536 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[36, 3331, 78448, 123974, 2673, 62209, 66041, 56504] avg=49152.0000 max=123974 max_rank=3 min=36 min_rank=0 imbalance_ratio=2.522257 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[49645, 49588, 49645, 49644, 45762, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45762 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74329 max_load_reduction_pct=0.599553 moved_to_replicas=138956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[81679, 44172, 43247, 63815, 90626, 2299, 65550, 1828] avg=49152.0000 max=90626 max_rank=4 min=1828 min_rank=7 imbalance_ratio=1.843791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=40982 max_load_reduction_pct=0.452210 moved_to_replicas=103094 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[95266, 30067, 69601, 7, 70156, 61242, 63658, 3219] avg=49152.0000 max=95266 max_rank=0 min=7 min_rank=3 imbalance_ratio=1.938192 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[49644, 49644, 49644, 45708, 49645, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=45621 max_load_reduction_pct=0.478880 moved_to_replicas=111703 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[64445, 65178, 3867, 49392, 62350, 88981, 3061, 55942] avg=49152.0000 max=88981 max_rank=5 min=3061 min_rank=6 imbalance_ratio=1.810323 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[49644, 49644, 46908, 49392, 49644, 49642, 48698, 49644] avg=49152.0000 max=49644 max_rank=0 min=46908 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=39337 max_load_reduction_pct=0.442083 moved_to_replicas=88676 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[2507, 119490, 51077, 19198, 157811, 706, 3, 42424] avg=49152.0000 max=157811 max_rank=4 min=3 min_rank=6 imbalance_ratio=3.210673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[47223, 49644, 49644, 48442, 49644, 49644, 49645, 49330] avg=49152.0000 max=49645 max_rank=6 min=47223 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=108166 max_load_reduction_pct=0.685415 moved_to_replicas=179446 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=23 total_expected_recv=1135536 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=52864 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=23 total_expected_recv=1133176 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55040 peak_demand=49647 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=23 total_expected_recv=1123706 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=65152 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=23 total_expected_recv=1134084 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=53120 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=23 total_expected_recv=1130625 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57984 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=23 total_expected_recv=1127312 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=61440 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=23 total_expected_recv=1125840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=63360 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=23 total_expected_recv=1133689 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56448 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 18:00:18] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 637.9 | throughput per GPU (TFLOP/s/GPU): 200.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.787342E+00 | loss scale: 1.0 | grad norm: 5.890 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=52224 expected_recv=48474 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=52224 expected_recv=48143 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=52224 expected_recv=48379 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=52224 expected_recv=48474 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=52224 expected_recv=48143 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=52224 expected_recv=48379 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=52224 expected_recv=46610 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=52224 expected_recv=46610 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=47104 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=52224 expected_recv=48741 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=52224 expected_recv=48741 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=52224 expected_recv=49209 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=52224 expected_recv=49209 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=52224 expected_recv=47479 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=52224 expected_recv=47479 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=48000 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=52224 expected_recv=48310 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=52224 expected_recv=48073 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=52224 expected_recv=48310 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=52224 expected_recv=48073 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=52224 expected_recv=48430 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=52224 expected_recv=48430 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=52224 expected_recv=49493 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=52224 expected_recv=49493 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=52224 expected_recv=45861 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=52224 expected_recv=48246 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=52224 expected_recv=45861 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=52224 expected_recv=48246 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=52224 expected_recv=47106 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=52224 expected_recv=48493 expected_overflow=0 exact_recv_per_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=52224 expected_recv=47106 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=52224 expected_recv=48493 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=47744 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=52224 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=52224 expected_recv=45707 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=46208 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=52224 expected_recv=48353 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=52224 expected_recv=48353 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=52224 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=52224 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=52224 expected_recv=46998 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=52224 expected_recv=46998 actual_valid_recv=47616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=47616 slack_slots=4608 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=52224 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=52224 expected_recv=45709 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=46080 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=52224 expected_recv=45711 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=52224 expected_recv=45711 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=46080 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=52224 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=52224 expected_recv=46818 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=52224 expected_recv=48535 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=52224 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=52224 expected_recv=46818 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=52224 expected_recv=48535 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=47488 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=52224 expected_recv=47508 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=52224 expected_recv=47844 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=52224 expected_recv=47508 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=52224 expected_recv=47844 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=52224 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=52224 expected_recv=45709 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=52224 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=52224 expected_recv=45709 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[50126, 50130, 45080, 42699, 47992, 44023, 52457, 60709] avg=49152.0000 max=60709 max_rank=7 min=42699 min_rank=3 imbalance_ratio=1.235128 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[49644, 49644, 48379, 49644, 48474, 48143, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48143 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=11065 max_load_reduction_pct=0.182263 moved_to_replicas=14846 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[46116, 55748, 63686, 29969, 46610, 58346, 51972, 40769] avg=49152.0000 max=63686 max_rank=2 min=29969 min_rank=3 imbalance_ratio=1.295695 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[48741, 49645, 49644, 49643, 46610, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=1 min=46610 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14041 max_load_reduction_pct=0.220472 moved_to_replicas=31176 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[34639, 58453, 47479, 41325, 82633, 43071, 30324, 55292] avg=49152.0000 max=82633 max_rank=4 min=30324 min_rank=6 imbalance_ratio=1.681173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[48310, 49643, 47479, 49645, 49642, 49209, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47479 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=32988 max_load_reduction_pct=0.399211 moved_to_replicas=47446 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[41640, 75784, 52413, 53011, 35772, 66313, 26345, 41938] avg=49152.0000 max=75784 max_rank=1 min=26345 min_rank=6 imbalance_ratio=1.541829 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[48430, 49644, 49645, 49644, 49644, 49643, 48493, 48073] avg=49152.0000 max=49645 max_rank=2 min=48073 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=26139 max_load_reduction_pct=0.344914 moved_to_replicas=48945 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[67005, 44641, 26752, 60994, 26861, 40505, 58869, 67589] avg=49152.0000 max=67589 max_rank=7 min=26752 min_rank=2 imbalance_ratio=1.375102 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[49644, 49643, 45861, 49643, 49645, 49493, 49644, 49643] avg=49152.0000 max=49645 max_rank=4 min=45861 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=17944 max_load_reduction_pct=0.265487 moved_to_replicas=55881 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[56241, 29348, 58816, 67062, 47104, 40786, 50786, 43073] avg=49152.0000 max=67062 max_rank=3 min=29348 min_rank=1 imbalance_ratio=1.364380 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[49643, 47106, 49644, 49644, 48246, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47106 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=17417 max_load_reduction_pct=0.259715 moved_to_replicas=34329 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[49669, 29771, 21186, 9940, 41101, 111540, 89489, 40520] avg=49152.0000 max=111540 max_rank=5 min=9940 min_rank=3 imbalance_ratio=2.269287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[49644, 49645, 45708, 49644, 49643, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=61895 max_load_reduction_pct=0.554913 moved_to_replicas=101766 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[83137, 21464, 29252, 96609, 26426, 58343, 59530, 18455] avg=49152.0000 max=96609 max_rank=3 min=18455 min_rank=7 imbalance_ratio=1.965515 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[49643, 49643, 49645, 49644, 45707, 49645, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46964 max_load_reduction_pct=0.486124 moved_to_replicas=99043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[179807, 17005, 36863, 7696, 67042, 2222, 34228, 48353] avg=49152.0000 max=179807 max_rank=0 min=2222 min_rank=5 imbalance_ratio=3.658183 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[49644, 49645, 49645, 49644, 49643, 49644, 46998, 48353] avg=49152.0000 max=49645 max_rank=1 min=46998 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=130162 max_load_reduction_pct=0.723898 moved_to_replicas=147561 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[122254, 28022, 13337, 7018, 104386, 31502, 6744, 79953] avg=49152.0000 max=122254 max_rank=0 min=6744 min_rank=6 imbalance_ratio=2.487264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[49644, 45708, 49644, 49646, 49643, 49645, 49643, 49643] avg=49152.0000 max=49646 max_rank=3 min=45708 min_rank=1 imbalance_ratio=1.010050 max_load_reduction=72608 max_load_reduction_pct=0.593911 moved_to_replicas=157661 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[2764, 25636, 1151, 65582, 55490, 87044, 7224, 148325] avg=49152.0000 max=148325 max_rank=7 min=1151 min_rank=2 imbalance_ratio=3.017680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[49644, 45707, 49644, 49644, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=5 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=98680 max_load_reduction_pct=0.665296 moved_to_replicas=157865 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[63321, 808, 77419, 33435, 79837, 28412, 85998, 23986] avg=49152.0000 max=85998 max_rank=6 min=808 min_rank=1 imbalance_ratio=1.749634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[49644, 49645, 49644, 45708, 49645, 49644, 49643, 49643] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=36353 max_load_reduction_pct=0.422719 moved_to_replicas=107999 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[5113, 40915, 59618, 136198, 61724, 29911, 40491, 19246] avg=49152.0000 max=136198 max_rank=3 min=5113 min_rank=0 imbalance_ratio=2.770955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[49644, 45708, 49645, 49645, 49645, 49643, 49643, 49643] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=86553 max_load_reduction_pct=0.635494 moved_to_replicas=108608 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[21081, 75754, 6618, 15894, 41910, 40434, 67666, 123859] avg=49152.0000 max=123859 max_rank=7 min=6618 min_rank=2 imbalance_ratio=2.519918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[45709, 49644, 49643, 49644, 49644, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=74214 max_load_reduction_pct=0.599181 moved_to_replicas=118347 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[76697, 12926, 2371, 1833, 30411, 197297, 56019, 15662] avg=49152.0000 max=197297 max_rank=5 min=1833 min_rank=3 imbalance_ratio=4.014018 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49643, 49644, 49644, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=147652 max_load_reduction_pct=0.748374 moved_to_replicas=181081 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[41505, 36484, 5911, 96973, 20494, 123858, 58144, 9847] avg=49152.0000 max=123858 max_rank=5 min=5911 min_rank=2 imbalance_ratio=2.519897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[49643, 49644, 49644, 49644, 45708, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=74213 max_load_reduction_pct=0.599178 moved_to_replicas=130043 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[50004, 121092, 13002, 119046, 62822, 2853, 2631, 21766] avg=49152.0000 max=121092 max_rank=1 min=2631 min_rank=6 imbalance_ratio=2.463623 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[49644, 49642, 45711, 49644, 49643, 49645, 49643, 49644] avg=49152.0000 max=49645 max_rank=5 min=45711 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=71447 max_load_reduction_pct=0.590022 moved_to_replicas=154388 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[5, 6862, 142114, 72308, 158436, 907, 5823, 6761] avg=49152.0000 max=158436 max_rank=4 min=5 min_rank=0 imbalance_ratio=3.223389 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49644, 49644, 49643, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=108791 max_load_reduction_pct=0.686656 moved_to_replicas=223926 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[8656, 7035, 95778, 125888, 8336, 40775, 84429, 22319] avg=49152.0000 max=125888 max_rank=3 min=7035 min_rank=1 imbalance_ratio=2.561198 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[49645, 49643, 49644, 49643, 49645, 45708, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=76243 max_load_reduction_pct=0.605642 moved_to_replicas=157163 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[122344, 5, 9858, 57273, 117829, 3631, 70496, 11780] avg=49152.0000 max=122344 max_rank=0 min=5 min_rank=1 imbalance_ratio=2.489095 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[49644, 49644, 48535, 49643, 49644, 46818, 49643, 49645] avg=49152.0000 max=49645 max_rank=7 min=46818 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=72699 max_load_reduction_pct=0.594218 moved_to_replicas=169366 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[68336, 12945, 91984, 0, 63035, 59304, 82187, 15425] avg=49152.0000 max=91984 max_rank=2 min=0 min_rank=3 imbalance_ratio=1.871419 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[49644, 47844, 49644, 49644, 49644, 49644, 49644, 47508] avg=49152.0000 max=49644 max_rank=0 min=47508 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=42340 max_load_reduction_pct=0.460297 moved_to_replicas=116626 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[62483, 68328, 4136, 60044, 52991, 122100, 2675, 20459] avg=49152.0000 max=122100 max_rank=5 min=2675 min_rank=6 imbalance_ratio=2.484131 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[49644, 49643, 49646, 49644, 49644, 49643, 49643, 45709] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=72454 max_load_reduction_pct=0.593399 moved_to_replicas=117726 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[2669, 78080, 3618, 66676, 168180, 7337, 800, 65856] avg=49152.0000 max=168180 max_rank=4 min=800 min_rank=6 imbalance_ratio=3.421631 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[49644, 49644, 45709, 49644, 49642, 49645, 49645, 49643] avg=49152.0000 max=49645 max_rank=5 min=45709 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=118535 max_load_reduction_pct=0.704810 moved_to_replicas=180216 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=23 total_expected_recv=1128332 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59904 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=23 total_expected_recv=1125663 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=63616 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=23 total_expected_recv=1128939 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58880 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=23 total_expected_recv=1121690 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=66816 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=23 total_expected_recv=1134077 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=55040 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=23 total_expected_recv=1132966 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=54272 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=23 total_expected_recv=1137876 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=51328 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=23 total_expected_recv=1134425 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56448 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 18:00:18] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 638.1 | throughput per GPU (TFLOP/s/GPU): 200.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.612353E+00 | loss scale: 1.0 | grad norm: 3.038 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=52224 expected_recv=48459 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=52224 expected_recv=47832 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=52224 expected_recv=48705 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=52224 expected_recv=48459 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=52224 expected_recv=47832 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=52224 expected_recv=48705 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=52224 expected_recv=48724 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=52224 expected_recv=47652 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=52224 expected_recv=48724 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=52224 expected_recv=47652 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=52224 expected_recv=48620 expected_overflow=0 exact_recv_per_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=52224 expected_recv=48620 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=52224 expected_recv=47554 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=52224 expected_recv=48324 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=52224 expected_recv=49121 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=52224 expected_recv=48324 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=52224 expected_recv=47554 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=52224 expected_recv=49121 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=52224 expected_recv=48467 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=52224 expected_recv=48467 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49280 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=52224 expected_recv=48460 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=52224 expected_recv=48069 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=52224 expected_recv=48460 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=52224 expected_recv=48069 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=52224 expected_recv=46417 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=52224 expected_recv=46417 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=47232 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=52224 expected_recv=48935 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=52224 expected_recv=48935 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=52224 expected_recv=48131 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=52224 expected_recv=48131 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=52224 expected_recv=47776 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=52224 expected_recv=49089 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=52224 expected_recv=47776 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=52224 expected_recv=49089 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=52224 expected_recv=48376 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=52224 expected_recv=48376 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=52224 expected_recv=46977 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=52224 expected_recv=46977 actual_valid_recv=47360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=47360 slack_slots=4864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=52224 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=52224 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=52224 expected_recv=47710 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=52224 expected_recv=47710 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=52224 expected_recv=47642 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=52224 expected_recv=47642 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=52224 expected_recv=47311 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=52224 expected_recv=47311 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=47872 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=52224 expected_recv=48394 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=52224 expected_recv=48041 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=52224 expected_recv=48394 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=52224 expected_recv=48041 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=52224 expected_recv=48212 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=52224 expected_recv=48391 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=52224 expected_recv=48212 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=52224 expected_recv=48391 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=52224 expected_recv=46300 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=52224 expected_recv=46300 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=46848 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=52224 expected_recv=49114 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=52224 expected_recv=49114 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=52224 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=52224 expected_recv=46236 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=52224 expected_recv=49052 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=52224 expected_recv=46236 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=47104 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=52224 expected_recv=49306 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=52224 expected_recv=47179 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=52224 expected_recv=49306 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=52224 expected_recv=47179 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=47744 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=52224 expected_recv=48510 expected_overflow=0 exact_recv_per_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=52224 expected_recv=48510 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=49280 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46080 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=46080 slack_slots=6144 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=52224 expected_recv=45706 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=52224 expected_recv=45706 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=52224 expected_recv=46443 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=52224 expected_recv=47938 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=52224 expected_recv=48908 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=52224 expected_recv=46443 actual_valid_recv=47104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=52224 expected_recv=47938 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=52224 expected_recv=48908 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=47104 slack_slots=5120 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=52224 expected_recv=47415 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=52224 expected_recv=47415 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=52224 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=52224 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[50671, 48705, 46089, 43130, 46745, 45271, 53886, 58719] avg=49152.0000 max=58719 max_rank=7 min=43130 min_rank=3 imbalance_ratio=1.194641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[49644, 48705, 49642, 49643, 48459, 47832, 49646, 49645] avg=49152.0000 max=49646 max_rank=6 min=47832 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=9073 max_load_reduction_pct=0.154516 moved_to_replicas=14344 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[45249, 53067, 63301, 28579, 53014, 62443, 48724, 38839] avg=49152.0000 max=63301 max_rank=2 min=28579 min_rank=3 imbalance_ratio=1.287862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[48620, 49645, 49644, 47652, 49643, 49644, 48724, 49644] avg=49152.0000 max=49645 max_rank=1 min=47652 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=13656 max_load_reduction_pct=0.215731 moved_to_replicas=33249 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[37101, 66962, 44390, 38426, 71468, 46529, 37671, 50669] avg=49152.0000 max=71468 max_rank=4 min=37101 min_rank=0 imbalance_ratio=1.454020 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[49644, 49643, 49121, 48324, 49643, 47554, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=47554 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21824 max_load_reduction_pct=0.305367 moved_to_replicas=40167 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[47687, 58023, 48069, 48460, 43392, 66140, 37657, 43788] avg=49152.0000 max=66140 max_rank=5 min=37657 min_rank=6 imbalance_ratio=1.345622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[49644, 49645, 48069, 48460, 48467, 49644, 49644, 49643] avg=49152.0000 max=49645 max_rank=1 min=48069 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=16495 max_load_reduction_pct=0.249395 moved_to_replicas=24875 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[62694, 43665, 25485, 51244, 41368, 43517, 55885, 69358] avg=49152.0000 max=69358 max_rank=7 min=25485 min_rank=2 imbalance_ratio=1.411092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46417, 48935, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=46417 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19714 max_load_reduction_pct=0.284235 moved_to_replicas=40605 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[55738, 38242, 44663, 59577, 61527, 38675, 48131, 46663] avg=49152.0000 max=61527 max_rank=4 min=38242 min_rank=1 imbalance_ratio=1.251770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 49089, 48131, 47776] avg=49152.0000 max=49644 max_rank=0 min=47776 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=11883 max_load_reduction_pct=0.193135 moved_to_replicas=27910 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[40287, 46977, 38149, 20037, 37862, 97829, 74640, 37435] avg=49152.0000 max=97829 max_rank=5 min=20037 min_rank=3 imbalance_ratio=1.990336 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[49644, 46977, 49644, 49643, 49644, 49644, 49644, 48376] avg=49152.0000 max=49644 max_rank=0 min=46977 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=48185 max_load_reduction_pct=0.492543 moved_to_replicas=73181 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[67635, 24792, 50578, 66151, 37580, 62371, 68492, 15617] avg=49152.0000 max=68492 max_rank=6 min=15617 min_rank=7 imbalance_ratio=1.393473 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[49644, 45708, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=45708 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=18848 max_load_reduction_pct=0.275185 moved_to_replicas=67007 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[158028, 21692, 34763, 20925, 62811, 16992, 35186, 42819] avg=49152.0000 max=158028 max_rank=0 min=16992 min_rank=5 imbalance_ratio=3.215088 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[49644, 49644, 49645, 49643, 49645, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=2 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=108383 max_load_reduction_pct=0.685847 moved_to_replicas=121551 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[105597, 29072, 21331, 12883, 78107, 33006, 20360, 92860] avg=49152.0000 max=105597 max_rank=0 min=12883 min_rank=3 imbalance_ratio=2.148376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[49645, 49644, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=0 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=55952 max_load_reduction_pct=0.529864 moved_to_replicas=127632 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[12753, 31526, 7558, 47642, 42895, 77968, 37506, 135368] avg=49152.0000 max=135368 max_rank=7 min=7558 min_rank=2 imbalance_ratio=2.754069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[49644, 49646, 49644, 47642, 47710, 49643, 49644, 49643] avg=49152.0000 max=49646 max_rank=1 min=47642 min_rank=3 imbalance_ratio=1.010050 max_load_reduction=85722 max_load_reduction_pct=0.633252 moved_to_replicas=114048 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[62676, 10333, 46569, 10688, 88569, 36614, 90456, 47311] avg=49152.0000 max=90456 max_rank=6 min=10333 min_rank=1 imbalance_ratio=1.840332 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[49644, 49645, 48041, 49643, 49644, 49643, 49645, 47311] avg=49152.0000 max=49645 max_rank=1 min=47311 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=40811 max_load_reduction_pct=0.451170 moved_to_replicas=92769 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[4302, 33098, 59673, 112034, 82189, 48212, 21316, 32392] avg=49152.0000 max=112034 max_rank=3 min=4302 min_rank=0 imbalance_ratio=2.279338 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[49644, 48391, 49644, 49643, 49645, 48212, 48394, 49643] avg=49152.0000 max=49645 max_rank=4 min=48212 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=62389 max_load_reduction_pct=0.556876 moved_to_replicas=104964 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[20719, 68407, 14347, 34851, 40436, 46300, 57986, 110170] avg=49152.0000 max=110170 max_rank=7 min=14347 min_rank=2 imbalance_ratio=2.241414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[49052, 49644, 49645, 49644, 49644, 46300, 49644, 49643] avg=49152.0000 max=49645 max_rank=2 min=46300 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=60525 max_load_reduction_pct=0.549378 moved_to_replicas=87631 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[57198, 16327, 13756, 2946, 71916, 161179, 49114, 20780] avg=49152.0000 max=161179 max_rank=5 min=2946 min_rank=3 imbalance_ratio=3.279195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49643, 49644, 49646, 49114, 46236] avg=49152.0000 max=49646 max_rank=5 min=46236 min_rank=7 imbalance_ratio=1.010050 max_load_reduction=111533 max_load_reduction_pct=0.691982 moved_to_replicas=141361 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[39089, 58604, 13764, 63613, 47179, 96019, 49306, 25642] avg=49152.0000 max=96019 max_rank=5 min=13764 min_rank=2 imbalance_ratio=1.953512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[48510, 49645, 49643, 49645, 47179, 49644, 49306, 49644] avg=49152.0000 max=49645 max_rank=1 min=47179 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=46374 max_load_reduction_pct=0.482967 moved_to_replicas=69304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[36530, 112675, 34085, 86808, 50155, 13427, 9237, 50299] avg=49152.0000 max=112675 max_rank=1 min=9237 min_rank=6 imbalance_ratio=2.292379 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[45708, 49642, 49644, 49645, 49644, 49645, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=63030 max_load_reduction_pct=0.559396 moved_to_replicas=101361 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[12706, 6117, 137177, 64300, 125823, 8753, 21500, 16840] avg=49152.0000 max=137177 max_rank=2 min=6117 min_rank=1 imbalance_ratio=2.790873 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49642, 49644, 45708, 49644] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=87532 max_load_reduction_pct=0.638095 moved_to_replicas=178368 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[21009, 12006, 94713, 110989, 28974, 27504, 62910, 35111] avg=49152.0000 max=110989 max_rank=3 min=12006 min_rank=1 imbalance_ratio=2.258077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[49644, 49644, 49644, 49646, 49644, 45706, 49644, 49644] avg=49152.0000 max=49646 max_rank=3 min=45706 min_rank=5 imbalance_ratio=1.010050 max_load_reduction=61343 max_load_reduction_pct=0.552694 moved_to_replicas=119680 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[137338, 5805, 16835, 28238, 82110, 11947, 70237, 40706] avg=49152.0000 max=137338 max_rank=0 min=5805 min_rank=1 imbalance_ratio=2.794149 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49644, 49644, 49643, 45708] avg=49152.0000 max=49645 max_rank=3 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=87693 max_load_reduction_pct=0.638520 moved_to_replicas=140753 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[63778, 32310, 58098, 834, 57312, 74586, 81073, 25225] avg=49152.0000 max=81073 max_rank=6 min=834 min_rank=3 imbalance_ratio=1.649434 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[49645, 46443, 49644, 48908, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=0 min=46443 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=31428 max_load_reduction_pct=0.387651 moved_to_replicas=86627 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[54037, 66029, 3961, 50827, 56995, 123325, 11405, 26637] avg=49152.0000 max=123325 max_rank=5 min=3961 min_rank=2 imbalance_ratio=2.509054 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49643, 49644, 47938, 47415] avg=49152.0000 max=49644 max_rank=0 min=47415 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=73681 max_load_reduction_pct=0.597454 moved_to_replicas=102993 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[29270, 73326, 16189, 58075, 135443, 14642, 8389, 57882] avg=49152.0000 max=135443 max_rank=4 min=8389 min_rank=6 imbalance_ratio=2.755595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[49644, 49644, 45707, 49644, 49645, 49643, 49645, 49644] avg=49152.0000 max=49645 max_rank=4 min=45707 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85798 max_load_reduction_pct=0.633462 moved_to_replicas=126150 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=23 total_expected_recv=1127684 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=60672 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=23 total_expected_recv=1131823 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56192 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=23 total_expected_recv=1129820 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58496 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=23 total_expected_recv=1127932 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59264 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=23 total_expected_recv=1122831 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=64640 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=23 total_expected_recv=1134171 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=53120 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=23 total_expected_recv=1134577 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=53120 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=23 total_expected_recv=1135130 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=52096 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 18:00:19] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 636.4 | throughput per GPU (TFLOP/s/GPU): 200.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.076241E+00 | loss scale: 1.0 | grad norm: 29.812 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=52224 expected_recv=48306 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=52224 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=52224 expected_recv=47764 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=52224 expected_recv=48306 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=52224 expected_recv=48927 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=52224 expected_recv=47764 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=52224 expected_recv=47614 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=52224 expected_recv=48714 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=52224 expected_recv=47614 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=52224 expected_recv=48714 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=49280 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=52224 expected_recv=48666 expected_overflow=0 exact_recv_per_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=52224 expected_recv=47654 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=52224 expected_recv=48666 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=52224 expected_recv=47654 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=52224 expected_recv=49611 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=52224 expected_recv=47698 expected_overflow=0 exact_recv_per_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=52224 expected_recv=49611 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=52224 expected_recv=47698 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=52224 expected_recv=48461 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=52224 expected_recv=48720 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=52224 expected_recv=48461 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=52224 expected_recv=48720 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=49280 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=52224 expected_recv=47632 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=52224 expected_recv=47848 expected_overflow=0 exact_recv_per_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=52224 expected_recv=47632 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=52224 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=52224 expected_recv=47848 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=52224 expected_recv=49149 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=52224 expected_recv=48213 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=52224 expected_recv=48213 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=52224 expected_recv=48135 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=52224 expected_recv=48135 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=52224 expected_recv=48146 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=52224 expected_recv=48146 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=52224 expected_recv=48715 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=52224 expected_recv=48715 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=52224 expected_recv=47724 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=52224 expected_recv=47724 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=52224 expected_recv=47628 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=52224 expected_recv=47628 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=52224 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=52224 expected_recv=45709 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=52224 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=52224 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=52224 expected_recv=47988 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=52224 expected_recv=47988 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=52224 expected_recv=47364 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=52224 expected_recv=47364 actual_valid_recv=47872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=47872 slack_slots=4352 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=52224 expected_recv=48636 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=52224 expected_recv=48636 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=49280 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=52224 expected_recv=46718 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=52224 expected_recv=46718 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=47232 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=52224 expected_recv=47571 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=52224 expected_recv=47571 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=52224 expected_recv=47782 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=52224 expected_recv=47782 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=52224 expected_recv=45804 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=52224 expected_recv=45804 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=52224 expected_recv=49547 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=52224 expected_recv=49547 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=52224 expected_recv=46774 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=52224 expected_recv=46774 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=47744 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=52224 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=52224 expected_recv=49322 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=52224 expected_recv=49130 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=52224 expected_recv=48900 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=52224 expected_recv=49130 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=52224 expected_recv=48900 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=52224 expected_recv=46221 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=52224 expected_recv=46221 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=46848 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=46592 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=52224 expected_recv=47700 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=52224 expected_recv=47700 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=52224 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=52224 expected_recv=47910 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=52224 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=52224 expected_recv=47910 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=52224 expected_recv=47651 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=52224 expected_recv=47651 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=52224 expected_recv=47796 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=52224 expected_recv=47796 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=52224 expected_recv=47442 expected_overflow=0 exact_recv_per_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=52224 expected_recv=47442 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=52224 expected_recv=47553 expected_overflow=0 exact_recv_per_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=52224 expected_recv=47553 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[50415, 47764, 46427, 43525, 47018, 46083, 53378, 58606] avg=49152.0000 max=58606 max_rank=7 min=43525 min_rank=3 imbalance_ratio=1.192342 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[49644, 47764, 49643, 49643, 48306, 48927, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=47764 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=8961 max_load_reduction_pct=0.152902 moved_to_replicas=13467 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[46171, 53499, 63677, 29165, 52140, 61001, 48714, 38849] avg=49152.0000 max=63677 max_rank=2 min=29165 min_rank=3 imbalance_ratio=1.295512 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[48666, 49645, 49644, 47614, 49645, 49643, 48714, 49645] avg=49152.0000 max=49645 max_rank=1 min=47614 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=14032 max_load_reduction_pct=0.220362 moved_to_replicas=31741 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[36218, 66279, 46178, 38874, 72792, 46556, 35804, 50515] avg=49152.0000 max=72792 max_rank=4 min=35804 min_rank=6 imbalance_ratio=1.480957 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[47698, 49645, 49643, 49644, 49644, 47654, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=47654 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=23147 max_load_reduction_pct=0.317988 moved_to_replicas=40654 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[47848, 61322, 47323, 50782, 43414, 63433, 34931, 44163] avg=49152.0000 max=63433 max_rank=5 min=34931 min_rank=6 imbalance_ratio=1.290548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[47848, 49644, 48461, 49644, 49644, 49644, 48720, 49611] avg=49152.0000 max=49644 max_rank=1 min=47848 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=13789 max_load_reduction_pct=0.217379 moved_to_replicas=26605 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[62864, 43749, 23535, 53136, 41276, 40207, 54665, 73784] avg=49152.0000 max=73784 max_rank=7 min=23535 min_rank=2 imbalance_ratio=1.501139 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[49644, 49149, 48213, 49645, 49645, 47632, 49644, 49644] avg=49152.0000 max=49645 max_rank=3 min=47632 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=24139 max_load_reduction_pct=0.327158 moved_to_replicas=45873 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[57708, 35767, 46435, 59363, 62023, 38416, 50668, 42836] avg=49152.0000 max=62023 max_rank=4 min=35767 min_rank=1 imbalance_ratio=1.261861 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[49644, 48146, 48715, 49644, 49644, 48135, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=48135 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12379 max_load_reduction_pct=0.199587 moved_to_replicas=31186 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[37084, 49886, 34330, 17360, 36730, 105889, 74102, 37835] avg=49152.0000 max=105889 max_rank=5 min=17360 min_rank=3 imbalance_ratio=2.154317 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[49644, 49644, 45708, 49644, 49644, 49644, 49645, 49643] avg=49152.0000 max=49645 max_rank=6 min=45708 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=56244 max_load_reduction_pct=0.531160 moved_to_replicas=80945 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[68204, 20680, 47628, 67126, 38290, 71465, 65843, 13980] avg=49152.0000 max=71465 max_rank=5 min=13980 min_rank=7 imbalance_ratio=1.453959 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[49644, 49645, 47628, 49644, 47724, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=1 min=47628 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21820 max_load_reduction_pct=0.305324 moved_to_replicas=74062 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[161595, 24547, 29168, 18359, 68911, 10722, 35364, 44550] avg=49152.0000 max=161595 max_rank=0 min=10722 min_rank=5 imbalance_ratio=3.287659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[49644, 49643, 49646, 49643, 49643, 49644, 45709, 49644] avg=49152.0000 max=49646 max_rank=2 min=45709 min_rank=6 imbalance_ratio=1.010050 max_load_reduction=111949 max_load_reduction_pct=0.692775 moved_to_replicas=131218 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[109639, 27093, 18489, 14079, 77831, 29832, 17357, 98896] avg=49152.0000 max=109639 max_rank=0 min=14079 min_rank=3 imbalance_ratio=2.230611 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[49644, 49644, 49644, 49645, 49645, 49643, 45707, 49644] avg=49152.0000 max=49645 max_rank=3 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=59994 max_load_reduction_pct=0.547196 moved_to_replicas=137434 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[4314, 27477, 2548, 47364, 45205, 92581, 33421, 140306] avg=49152.0000 max=140306 max_rank=7 min=2548 min_rank=2 imbalance_ratio=2.854533 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[49644, 49644, 47988, 47364, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=0 min=47364 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=90662 max_load_reduction_pct=0.646173 moved_to_replicas=133599 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[52644, 3419, 38520, 8871, 99082, 18350, 106965, 65365] avg=49152.0000 max=106965 max_rank=6 min=3419 min_rank=1 imbalance_ratio=2.176208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[49644, 48636, 49644, 46718, 49644, 49643, 49643, 49644] avg=49152.0000 max=49644 max_rank=0 min=46718 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=57321 max_load_reduction_pct=0.535886 moved_to_replicas=125480 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[2671, 38545, 61355, 112196, 95249, 34454, 21034, 27712] avg=49152.0000 max=112196 max_rank=3 min=2671 min_rank=0 imbalance_ratio=2.282633 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[49645, 49645, 49643, 49645, 49644, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=62551 max_load_reduction_pct=0.557515 moved_to_replicas=119868 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[28199, 50884, 18531, 45568, 30789, 46542, 55406, 117297] avg=49152.0000 max=117297 max_rank=7 min=18531 min_rank=2 imbalance_ratio=2.386414 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 47571, 47782, 49644, 49643] avg=49152.0000 max=49644 max_rank=0 min=47571 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=67653 max_load_reduction_pct=0.576767 moved_to_replicas=74655 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[58077, 17771, 4589, 2151, 85022, 165326, 41115, 19165] avg=49152.0000 max=165326 max_rank=5 min=2151 min_rank=3 imbalance_ratio=3.363566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49645, 49644, 49643, 49547, 45804] avg=49152.0000 max=49645 max_rank=0 min=45804 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=115681 max_load_reduction_pct=0.699715 moved_to_replicas=159493 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[53835, 66835, 24008, 55432, 36726, 74536, 53904, 27940] avg=49152.0000 max=74536 max_rank=5 min=24008 min_rank=2 imbalance_ratio=1.516439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[49644, 49644, 48900, 49644, 46774, 49644, 49644, 49322] avg=49152.0000 max=49644 max_rank=0 min=46774 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=24892 max_load_reduction_pct=0.333959 moved_to_replicas=56322 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[51891, 119725, 25101, 78265, 49130, 6638, 5815, 56651] avg=49152.0000 max=119725 max_rank=1 min=5815 min_rank=6 imbalance_ratio=2.435811 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49130, 46221, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46221 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=70080 max_load_reduction_pct=0.585341 moved_to_replicas=107956 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[5076, 11479, 147411, 72927, 123693, 2724, 14473, 15433] avg=49152.0000 max=147411 max_rank=2 min=2724 min_rank=5 imbalance_ratio=2.999084 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[49645, 49645, 49644, 49645, 49643, 49643, 49643, 45708] avg=49152.0000 max=49645 max_rank=0 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=97766 max_load_reduction_pct=0.663221 moved_to_replicas=195099 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[25889, 18762, 97824, 117150, 26220, 10751, 63165, 33455] avg=49152.0000 max=117150 max_rank=3 min=10751 min_rank=5 imbalance_ratio=2.383423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[45708, 49645, 49644, 49644, 49644, 49643, 49644, 49644] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=67505 max_load_reduction_pct=0.576227 moved_to_replicas=129207 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[164829, 2298, 12598, 17030, 80017, 11658, 55018, 49768] avg=49152.0000 max=164829 max_rank=0 min=2298 min_rank=1 imbalance_ratio=3.353455 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[49645, 49644, 49643, 45707, 49645, 49644, 49644, 49644] avg=49152.0000 max=49645 max_rank=0 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=115184 max_load_reduction_pct=0.698809 moved_to_replicas=151056 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[63730, 55977, 43109, 588, 40157, 60944, 82622, 46089] avg=49152.0000 max=82622 max_rank=6 min=588 min_rank=3 imbalance_ratio=1.680949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[49645, 49644, 49644, 47651, 49644, 49644, 49644, 47700] avg=49152.0000 max=49645 max_rank=0 min=47651 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=32977 max_load_reduction_pct=0.399131 moved_to_replicas=64697 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[45643, 73046, 7160, 56538, 47910, 140302, 7530, 15087] avg=49152.0000 max=140302 max_rank=5 min=7160 min_rank=2 imbalance_ratio=2.854451 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[47442, 49643, 49645, 49644, 47910, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=2 min=47442 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=90657 max_load_reduction_pct=0.646156 moved_to_replicas=120954 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[9891, 66981, 5560, 63771, 126177, 37686, 19756, 63394] avg=49152.0000 max=126177 max_rank=4 min=5560 min_rank=2 imbalance_ratio=2.567078 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[47553, 49644, 49644, 49645, 49646, 49644, 47796, 49644] avg=49152.0000 max=49646 max_rank=4 min=47553 min_rank=0 imbalance_ratio=1.010050 max_load_reduction=76531 max_load_reduction_pct=0.606537 moved_to_replicas=121747 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=23 total_expected_recv=1131367 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56192 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=23 total_expected_recv=1130140 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58624 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=23 total_expected_recv=1136935 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=51200 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=23 total_expected_recv=1130291 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57344 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=23 total_expected_recv=1127801 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59520 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=23 total_expected_recv=1129916 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57088 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=23 total_expected_recv=1128650 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59136 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=23 total_expected_recv=1128868 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=58496 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 18:00:20] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 638.9 | throughput per GPU (TFLOP/s/GPU): 199.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.639501E+00 | loss scale: 1.0 | grad norm: 64.594 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=52224 expected_recv=48125 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=52224 expected_recv=48853 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=52224 expected_recv=48695 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=52224 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=52224 expected_recv=48125 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=52224 expected_recv=48853 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=52224 expected_recv=48695 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=52224 expected_recv=48967 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=48896 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49408 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49280 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=52224 expected_recv=47633 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=52224 expected_recv=47633 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=48000 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=52224 expected_recv=47814 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=52224 expected_recv=49549 expected_overflow=0 exact_recv_per_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=52224 expected_recv=47814 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=52224 expected_recv=49549 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=52224 expected_recv=46904 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=52224 expected_recv=46904 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=47488 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=52224 expected_recv=48449 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=52224 expected_recv=48449 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=52224 expected_recv=48159 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=52224 expected_recv=48159 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=52224 expected_recv=47193 expected_overflow=0 exact_recv_per_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=52224 expected_recv=47193 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=52224 valid_num_tokens=47744 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=52224 expected_recv=48491 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=52224 expected_recv=48665 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=52224 expected_recv=48491 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=52224 expected_recv=48665 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=49152 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=52224 expected_recv=47840 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=52224 expected_recv=47840 actual_valid_recv=48640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=48640 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=52224 expected_recv=49452 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=52224 expected_recv=49452 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=52224 expected_recv=48853 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=52224 expected_recv=48853 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=52224 expected_recv=46691 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=52224 expected_recv=46691 actual_valid_recv=47488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=47488 slack_slots=4736 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=52224 expected_recv=47665 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=52224 expected_recv=47665 actual_valid_recv=48256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=48256 slack_slots=3968 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=52224 expected_recv=47687 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=52224 expected_recv=47687 actual_valid_recv=48128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=48128 slack_slots=4096 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=52224 expected_recv=47882 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=52224 expected_recv=47882 actual_valid_recv=48512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=48512 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=52224 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=52224 expected_recv=47965 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=52224 expected_recv=49149 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=52224 expected_recv=47965 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=49664 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=52224 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=52224 expected_recv=45707 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=52224 expected_recv=49036 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=52224 expected_recv=49036 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=49536 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=52224 expected_recv=46579 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=52224 expected_recv=46579 actual_valid_recv=47232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=47232 slack_slots=4992 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=52224 expected_recv=49381 expected_overflow=0 exact_recv_per_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=52224 expected_recv=49381 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=52224 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=52224 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=52224 expected_recv=48043 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=52224 expected_recv=48043 actual_valid_recv=48768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=48768 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=52224 expected_recv=47497 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=52224 expected_recv=49456 expected_overflow=0 exact_recv_per_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=52224 expected_recv=45706 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=52224 expected_recv=47497 actual_valid_recv=48000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=52224 expected_recv=49456 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=52224 expected_recv=45706 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=48000 slack_slots=4224 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=46592 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=52224 expected_recv=46829 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=52224 expected_recv=46829 actual_valid_recv=47744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=47744 slack_slots=4480 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=52224 expected_recv=48523 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=52224 expected_recv=48523 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=49024 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=52224 expected_recv=47705 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=52224 expected_recv=47646 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=52224 expected_recv=47705 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=52224 expected_recv=47646 actual_valid_recv=48384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=48384 slack_slots=3840 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=52224 expected_recv=45709 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=52224 expected_recv=45709 actual_valid_recv=46208 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=46208 slack_slots=6016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=16 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=49920 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=17 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=52224 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=52224 expected_recv=48989 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=49792 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50560 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=52224 expected_recv=46365 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=18 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50560 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=52224 expected_recv=46365 actual_valid_recv=46848 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=46848 slack_slots=5376 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46592 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=19 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=46592 slack_slots=5632 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=52224 expected_recv=49646 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=52224 expected_recv=49646 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50432 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=52224 expected_recv=45707 expected_overflow=0 exact_recv_per_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50432 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=52224 expected_recv=45707 actual_valid_recv=46464 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=20 calls=1 received_num_tokens=52224 valid_num_tokens=46464 slack_slots=5760 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=52224 expected_recv=49642 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=52224 expected_recv=49642 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=52224 expected_recv=49645 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=52224 expected_recv=49643 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=52224 expected_recv=49645 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=52224 expected_recv=45708 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=21 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=52224 expected_recv=49643 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50176 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=52224 expected_recv=45708 actual_valid_recv=46336 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50048 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=46336 slack_slots=5888 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=52224 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=52224 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=22 calls=1 received_num_tokens=52224 valid_num_tokens=50304 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[50034, 48967, 44735, 44088, 47735, 44544, 53604, 59509] avg=49152.0000 max=59509 max_rank=7 min=44088 min_rank=3 imbalance_ratio=1.210714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[49644, 48967, 48695, 49643, 48125, 48853, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=48125 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=9864 max_load_reduction_pct=0.165756 moved_to_replicas=14215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[45683, 53510, 63951, 31590, 47633, 58870, 50714, 41265] avg=49152.0000 max=63951 max_rank=2 min=31590 min_rank=3 imbalance_ratio=1.301086 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[49549, 49644, 49644, 47814, 47633, 49643, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47633 min_rank=4 imbalance_ratio=1.010030 max_load_reduction=14306 max_load_reduction_pct=0.223703 moved_to_replicas=28469 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[33629, 62755, 49716, 44512, 72355, 45779, 33773, 50697] avg=49152.0000 max=72355 max_rank=4 min=33629 min_rank=0 imbalance_ratio=1.472066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[49644, 49644, 49644, 48449, 49645, 46904, 49642, 49644] avg=49152.0000 max=49645 max_rank=4 min=46904 min_rank=5 imbalance_ratio=1.010030 max_load_reduction=22710 max_load_reduction_pct=0.313869 moved_to_replicas=36947 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[46913, 69628, 48159, 56147, 42357, 57255, 30297, 42460] avg=49152.0000 max=69628 max_rank=1 min=30297 min_rank=6 imbalance_ratio=1.416585 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[47193, 49644, 48159, 49644, 49644, 49645, 49644, 49643] avg=49152.0000 max=49645 max_rank=5 min=47193 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=19983 max_load_reduction_pct=0.286997 moved_to_replicas=34098 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[64044, 50879, 24364, 58315, 38341, 34232, 51714, 71327] avg=49152.0000 max=71327 max_rank=7 min=24364 min_rank=2 imbalance_ratio=1.451152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[49645, 49644, 47840, 49645, 48491, 48665, 49643, 49643] avg=49152.0000 max=49645 max_rank=0 min=47840 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=21682 max_load_reduction_pct=0.303980 moved_to_replicas=48059 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[55779, 36560, 52613, 56807, 57331, 35629, 56073, 42424] avg=49152.0000 max=57331 max_rank=4 min=35629 min_rank=5 imbalance_ratio=1.166402 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[49644, 46691, 49644, 49645, 49643, 49452, 49644, 48853] avg=49152.0000 max=49645 max_rank=3 min=46691 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=7686 max_load_reduction_pct=0.134064 moved_to_replicas=30383 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[40855, 41412, 27625, 14900, 38722, 110580, 77749, 41373] avg=49152.0000 max=110580 max_rank=5 min=14900 min_rank=3 imbalance_ratio=2.249756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[49644, 49644, 49646, 49643, 45708, 49643, 49644, 49644] avg=49152.0000 max=49646 max_rank=2 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=60934 max_load_reduction_pct=0.551040 moved_to_replicas=89041 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[84846, 22602, 34525, 73257, 37497, 71353, 47665, 21471] avg=49152.0000 max=84846 max_rank=0 min=21471 min_rank=7 imbalance_ratio=1.726196 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[49644, 49644, 47687, 49644, 49644, 49644, 47665, 49644] avg=49152.0000 max=49644 max_rank=0 min=47665 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=35202 max_load_reduction_pct=0.414893 moved_to_replicas=80524 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[144579, 29766, 23368, 15722, 74287, 10061, 43117, 52316] avg=49152.0000 max=144579 max_rank=0 min=10061 min_rank=5 imbalance_ratio=2.941467 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[49643, 49644, 49149, 47965, 49644, 49644, 47882, 49645] avg=49152.0000 max=49645 max_rank=7 min=47882 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=94934 max_load_reduction_pct=0.656624 moved_to_replicas=122250 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[103954, 27128, 18399, 14653, 87177, 34132, 16283, 91490] avg=49152.0000 max=103954 max_rank=0 min=14653 min_rank=3 imbalance_ratio=2.114950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[49644, 45707, 49645, 49644, 49645, 49644, 49643, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=1 imbalance_ratio=1.010030 max_load_reduction=54309 max_load_reduction_pct=0.522433 moved_to_replicas=133689 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[4169, 31386, 2703, 43707, 49036, 104209, 23336, 134670] avg=49152.0000 max=134670 max_rank=7 min=2703 min_rank=2 imbalance_ratio=2.739868 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[49381, 49643, 46579, 49644, 49036, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=6 min=46579 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=85025 max_load_reduction_pct=0.631358 moved_to_replicas=139591 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[38116, 2914, 41732, 8154, 104773, 22837, 104658, 70032] avg=49152.0000 max=104773 max_rank=4 min=2914 min_rank=1 imbalance_ratio=2.131612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[45708, 49644, 49644, 49644, 49644, 49644, 49644, 49644] avg=49152.0000 max=49644 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55129 max_load_reduction_pct=0.526176 moved_to_replicas=130531 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[2844, 36738, 60570, 108559, 105486, 26192, 26317, 26510] avg=49152.0000 max=108559 max_rank=3 min=2844 min_rank=0 imbalance_ratio=2.208639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[49644, 49643, 49645, 49645, 49644, 49643, 45707, 49645] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=58914 max_load_reduction_pct=0.542691 moved_to_replicas=125683 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[29776, 43798, 16492, 47002, 31846, 50200, 55985, 118117] avg=49152.0000 max=118117 max_rank=7 min=16492 min_rank=2 imbalance_ratio=2.403097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[49456, 49644, 49643, 47497, 48043, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=7 min=47497 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=68472 max_load_reduction_pct=0.579696 moved_to_replicas=75370 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[72683, 16278, 5780, 2543, 79004, 159477, 33460, 23991] avg=49152.0000 max=159477 max_rank=5 min=2543 min_rank=3 imbalance_ratio=3.244568 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[49645, 49644, 49644, 49644, 49644, 49645, 49644, 45706] avg=49152.0000 max=49645 max_rank=0 min=45706 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=109832 max_load_reduction_pct=0.688701 moved_to_replicas=162232 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[69123, 60743, 34399, 52175, 34608, 50980, 53764, 37424] avg=49152.0000 max=69123 max_rank=0 min=34399 min_rank=2 imbalance_ratio=1.406311 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 46829, 49644, 49644, 48523] avg=49152.0000 max=49644 max_rank=0 min=46829 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=19479 max_load_reduction_pct=0.281802 moved_to_replicas=38565 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[62984, 125040, 7015, 70461, 53952, 9240, 8090, 56434] avg=49152.0000 max=125040 max_rank=1 min=7015 min_rank=2 imbalance_ratio=2.543945 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[49644, 49644, 49644, 49644, 49644, 47705, 47646, 49645] avg=49152.0000 max=49645 max_rank=7 min=47646 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=75395 max_load_reduction_pct=0.602967 moved_to_replicas=120651 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[5346, 24311, 134163, 80479, 110605, 3052, 15208, 20052] avg=49152.0000 max=134163 max_rank=2 min=3052 min_rank=5 imbalance_ratio=2.729553 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[49644, 49645, 49643, 49643, 49644, 49644, 45709, 49644] avg=49152.0000 max=49645 max_rank=1 min=45709 min_rank=6 imbalance_ratio=1.010030 max_load_reduction=84518 max_load_reduction_pct=0.629965 moved_to_replicas=176315 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[32874, 21025, 84147, 111381, 35694, 11701, 66210, 30184] avg=49152.0000 max=111381 max_rank=3 min=11701 min_rank=5 imbalance_ratio=2.266052 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[49644, 49643, 49644, 49644, 49644, 49645, 49644, 45708] avg=49152.0000 max=49645 max_rank=5 min=45708 min_rank=7 imbalance_ratio=1.010030 max_load_reduction=61736 max_load_reduction_pct=0.554278 moved_to_replicas=112806 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[168970, 2875, 21337, 14295, 81831, 14967, 33723, 55218] avg=49152.0000 max=168970 max_rank=0 min=2875 min_rank=1 imbalance_ratio=3.437703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[49644, 49643, 46365, 49645, 49644, 49642, 48989, 49644] avg=49152.0000 max=49645 max_rank=3 min=46365 min_rank=2 imbalance_ratio=1.010030 max_load_reduction=119325 max_load_reduction_pct=0.706190 moved_to_replicas=157087 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[69828, 68082, 20796, 1054, 23106, 60714, 89901, 59735] avg=49152.0000 max=89901 max_rank=6 min=1054 min_rank=3 imbalance_ratio=1.829041 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[49643, 49644, 49645, 45707, 49644, 49644, 49645, 49644] avg=49152.0000 max=49645 max_rank=2 min=45707 min_rank=3 imbalance_ratio=1.010030 max_load_reduction=40256 max_load_reduction_pct=0.447781 moved_to_replicas=100040 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[13539, 95670, 11195, 66143, 29485, 152588, 9426, 15170] avg=49152.0000 max=152588 max_rank=5 min=9426 min_rank=6 imbalance_ratio=3.104411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[49642, 49644, 49644, 49645, 45708, 49646, 49643, 49644] avg=49152.0000 max=49646 max_rank=5 min=45708 min_rank=4 imbalance_ratio=1.010050 max_load_reduction=102942 max_load_reduction_pct=0.674640 moved_to_replicas=165469 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[12482, 62523, 7529, 59017, 118936, 45303, 19909, 67517] avg=49152.0000 max=118936 max_rank=4 min=7529 min_rank=2 imbalance_ratio=2.419759 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[45708, 49645, 49643, 49644, 49643, 49644, 49644, 49645] avg=49152.0000 max=49645 max_rank=1 min=45708 min_rank=0 imbalance_ratio=1.010030 max_load_reduction=69291 max_load_reduction_pct=0.582591 moved_to_replicas=109417 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=23 total_expected_recv=1124233 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=62336 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=23 total_expected_recv=1127543 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=60416 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=23 total_expected_recv=1134243 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=52864 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=23 total_expected_recv=1135171 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=52096 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=23 total_expected_recv=1132031 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56320 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=23 total_expected_recv=1130941 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=56576 peak_demand=49645 required_capacity=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=23 total_expected_recv=1128780 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=59008 peak_demand=49646 required_capacity=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=23 total_expected_recv=1131026 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=57088 peak_demand=49645 required_capacity=1.010030 + [2026-08-26 18:00:20] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 638.1 | throughput per GPU (TFLOP/s/GPU): 199.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.624613E+00 | loss scale: 1.0 | grad norm: 71.371 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50560 | min safe capacity: 1.0286 | +[after training is done] datetime: 2026-08-26 18:00:20 +[rank0]:[W826 18:00:21.942392195 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 18:00:23.245372473 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 18:00:23.246658260 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 18:00:23.247118288 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 18:00:23.247417529 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 18:00:23.615652961 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 18:00:23.617088588 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 18:00:23.619570675 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 18:00:24.615000 439248 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 18:00:24.615000 439248 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 439348 closing signal SIGTERM +W0826 18:00:24.616000 439248 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 439349 closing signal SIGTERM +W0826 18:00:24.618000 439248 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 439350 closing signal SIGTERM +W0826 18:00:24.619000 439248 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 439351 closing signal SIGTERM +W0826 18:00:24.621000 439248 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 439352 closing signal SIGTERM +W0826 18:00:24.622000 439248 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 439353 closing signal SIGTERM +W0826 18:00:24.624000 439248 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 439354 closing signal SIGTERM +W0826 18:00:24.625000 439248 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 439355 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 439248 got signal: 15 diff --git a/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv b/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv index be74274b498a0ed9b8595645a17b81bf99463c0b..ca42e201a9f0856b744faf05b5bb4b995445f589 100644 --- a/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv +++ b/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/mem_samples.csv @@ -14,7 +14,7 @@ 5, 0 6, 0 7, 0 -0, 1 +0, 2 1, 1 2, 1 3, 1 @@ -39,21 +39,21 @@ 6, 4 7, 4 0, 4 -1, 629 -2, 629 -3, 47 -4, 629 -5, 629 -6, 629 -7, 629 -0, 999 -1, 999 -2, 995 -3, 989 -4, 993 -5, 973 -6, 967 -7, 997 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 0, 1955 1, 1955 2, 1955 @@ -62,22 +62,22 @@ 5, 1955 6, 1955 7, 1955 -0, 2389 -1, 2387 -2, 2405 -3, 2401 -4, 2419 -5, 2401 -6, 2393 -7, 2395 -0, 25415 -1, 25393 -2, 25437 -3, 25371 -4, 25371 -5, 25415 -6, 25415 -7, 25481 +0, 2325 +1, 2345 +2, 2363 +3, 2339 +4, 2337 +5, 2351 +6, 2331 +7, 2357 +0, 10379 +1, 10379 +2, 10379 +3, 10379 +4, 10379 +5, 16131 +6, 10373 +7, 10379 0, 28963 1, 29051 2, 28897 @@ -86,14 +86,14 @@ 5, 28897 6, 29029 7, 29051 -0, 39003 -1, 39107 -2, 38941 -3, 38911 -4, 38911 -5, 38939 -6, 39071 -7, 39065 +0, 39061 +1, 39165 +2, 39015 +3, 39015 +4, 39009 +5, 38985 +6, 39123 +7, 39163 0, 39771 1, 39855 2, 39705 @@ -126,14 +126,14 @@ 5, 39711 6, 39847 7, 39865 -0, 111669 -1, 111785 -2, 111681 -3, 111673 -4, 111685 -5, 111675 -6, 111801 -7, 111805 +0, 111737 +1, 111833 +2, 111747 +3, 111659 +4, 111753 +5, 111691 +6, 111825 +7, 111833 0, 112149 1, 112235 2, 112149 @@ -150,30 +150,22 @@ 5, 112085 6, 112221 7, 112239 -0, 114445 -1, 112503 -2, 114451 -3, 114383 -4, 114429 -5, 114345 -6, 114471 -7, 112505 -0, 115241 -1, 113277 -2, 115239 -3, 115175 -4, 115239 -5, 115171 -6, 115307 -7, 113277 -0, 115861 -1, 113909 -2, 115895 -3, 115835 -4, 115863 -5, 115811 -6, 115889 -7, 113869 +0, 114419 +1, 112455 +2, 114449 +3, 114351 +4, 114425 +5, 114379 +6, 114489 +7, 112535 +0, 115415 +1, 113455 +2, 115413 +3, 115385 +4, 115397 +5, 115307 +6, 115505 +7, 113477 0, 127845 1, 127923 2, 129893 @@ -198,35 +190,43 @@ 5, 129817 6, 129959 7, 129971 -0, 127283 -1, 127623 -2, 129593 -3, 129525 -4, 129589 -5, 129517 -6, 129659 -7, 129607 -0, 123747 -1, 124407 -2, 126439 -3, 126115 -4, 126373 -5, 126427 -6, 126831 -7, 126517 -0, 122387 -1, 122727 -2, 124633 -3, 124367 -4, 124373 -5, 124685 -6, 125409 -7, 125095 -0, 1 -1, 3 +0, 127845 +1, 127923 +2, 129893 +3, 129825 +4, 129889 +5, 129817 +6, 129959 +7, 129971 +0, 126899 +1, 126977 +2, 128947 +3, 128879 +4, 128943 +5, 128871 +6, 129013 +7, 129025 +0, 123223 +1, 123761 +2, 125731 +3, 125663 +4, 125727 +5, 126043 +6, 125797 +7, 125421 +0, 4 +1, 122053 2, 4 3, 4 -4, 4 -5, 68 -6, 124761 -7, 324 +4, 124017 +5, 124075 +6, 4 +7, 4 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log b/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log index 8289d65c55c2a3d585fddf5341c4519f54e06e45..14f1ba17edd29f8fe9698ceba210d793721e6c3d 100644 --- a/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log +++ b/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/train.log @@ -1,4 +1,4 @@ -[16:48:22] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[18:00:29] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) ============================================================== DSV2_LITE_L24 / ultraep (slots/rank=8 policy=matched) E=64 K=6 H'=1408 m=8 @@ -7,18 +7,10 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-UltraEP @ eaf829e7 ============================================================== -W0826 16:48:23.555000 270643 torch/distributed/run.py:766] -W0826 16:48:23.555000 270643 torch/distributed/run.py:766] ***************************************** -W0826 16:48:23.555000 270643 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 16:48:23.555000 270643 torch/distributed/run.py:766] ***************************************** -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +W0826 18:00:30.426000 442964 torch/distributed/run.py:766] +W0826 18:00:30.426000 442964 torch/distributed/run.py:766] ***************************************** +W0826 18:00:30.426000 442964 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 18:00:30.426000 442964 torch/distributed/run.py:766] ***************************************** using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer Number of virtual stages per pipeline stage: None @@ -100,6 +92,8 @@ using torch.bfloat16 for parameters ... data_per_class_fraction ......................... 1.0 data_sharding ................................... True dataloader_type ................................. single +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") ddp_average_in_collective ....................... False ddp_bucket_size ................................. None ddp_num_buckets ................................. None @@ -573,6 +567,7 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... @@ -582,52 +577,57 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank [Gloo] Rank 13 is connected to 6 is connected to 7 is connected to 7 peer ranks. 7 peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : 7 -7 - -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 7 is connected to 57 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : -7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 1 is connected to 07 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : -7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 4 is connected to 07 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +[Gloo] Rank 73 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to [Gloo] Rank 7 peer ranks. 7Expected number of connected peer ranks is : is connected to 77 - peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 57 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 -[rank4]:[W826 16:48:41.844756902 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 16:48:41.844777838 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 16:48:41.844830787 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 16:48:41.844902808 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 16:48:41.845133202 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 16:48:41.848764929 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 16:48:41.848793749 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank4]:[W826 18:00:48.628010848 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 18:00:48.628038927 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 18:00:48.630416691 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 18:00:48.630518915 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 18:00:48.630566785 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 18:00:48.631861391 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 18:00:48.632100431 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.033 seconds +>>> done with dataset index builder. Compilation time: 0.034 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 16:48:41.004678275 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.703 seconds -time to initialize megatron (seconds): 6.889 -[after megatron is initialized] datetime: 2026-08-26 16:48:47 +[rank0]:[W826 18:00:48.781377286 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.520 seconds +time to initialize megatron (seconds): 6.870 +[after megatron is initialized] datetime: 2026-08-26 18:00:54 building GPT model ... /workspace/Megatron-UltraEP/megatron/core/transformer/transformer_config.py:1535: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. warnings.warn( @@ -687,23 +687,23 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi Params for bucket 1 (103022592 elements, 103022592 padded size): module.output_layer.weight Params for bucket 2 (48503296 elements, 48503296 padded size): - module.decoder.final_layernorm.weight - module.decoder.layers.23.mlp.router.weight - module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.23.self_attention.linear_q_proj.weight - module.decoder.layers.23.input_layernorm.weight - module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight module.decoder.layers.23.pre_mlp_layernorm.weight module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight module.decoder.layers.23.self_attention.linear_kv_up_proj.weight module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.23.self_attention.linear_kv_down_proj.weight module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.22.self_attention.linear_kv_down_proj.weight module.decoder.layers.22.self_attention.linear_kv_up_proj.weight - module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.22.self_attention.linear_q_proj.weight module.decoder.layers.22.self_attention.linear_proj.weight module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight @@ -725,83 +725,83 @@ Params for bucket 4 (41163264 elements, 41163264 padded size): module.decoder.layers.20.self_attention.linear_kv_down_proj.weight module.decoder.layers.20.self_attention.linear_q_proj.weight module.decoder.layers.20.self_attention.linear_proj.weight - module.decoder.layers.20.input_layernorm.weight module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight module.decoder.layers.21.input_layernorm.weight module.decoder.layers.20.mlp.router.weight module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.input_layernorm.weight module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight Params for bucket 5 (42734080 elements, 42734080 padded size): module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.19.self_attention.linear_kv_up_proj.weight module.decoder.layers.19.self_attention.linear_kv_down_proj.weight module.decoder.layers.19.self_attention.linear_q_proj.weight module.decoder.layers.19.self_attention.linear_proj.weight module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.19.mlp.router.weight - module.decoder.layers.19.pre_mlp_layernorm.weight - module.decoder.layers.19.input_layernorm.weight - module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight Params for bucket 6 (40901632 elements, 40901632 padded size): - module.decoder.layers.18.self_attention.linear_kv_down_proj.weight - module.decoder.layers.18.self_attention.linear_kv_up_proj.weight - module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.18.self_attention.linear_q_proj.weight - module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight module.decoder.layers.17.self_attention.linear_kv_up_proj.weight module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.17.self_attention.linear_kv_down_proj.weight module.decoder.layers.17.self_attention.linear_q_proj.weight - module.decoder.layers.18.mlp.router.weight - module.decoder.layers.18.pre_mlp_layernorm.weight - module.decoder.layers.18.input_layernorm.weight - module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_proj.weight module.decoder.layers.17.mlp.router.weight module.decoder.layers.17.pre_mlp_layernorm.weight Params for bucket 7 (41163264 elements, 41163264 padded size): module.decoder.layers.17.self_attention.linear_proj.weight - module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight module.decoder.layers.16.self_attention.linear_kv_up_proj.weight module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.16.self_attention.linear_kv_down_proj.weight module.decoder.layers.16.self_attention.linear_q_proj.weight module.decoder.layers.16.self_attention.linear_proj.weight - module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.input_layernorm.weight module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight module.decoder.layers.16.pre_mlp_layernorm.weight - module.decoder.layers.16.input_layernorm.weight module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight Params for bucket 8 (42734080 elements, 42734080 padded size): module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.15.pre_mlp_layernorm.weight - module.decoder.layers.15.mlp.router.weight - module.decoder.layers.15.input_layernorm.weight - module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight module.decoder.layers.15.self_attention.linear_kv_up_proj.weight module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.15.self_attention.linear_kv_down_proj.weight module.decoder.layers.15.self_attention.linear_q_proj.weight module.decoder.layers.15.self_attention.linear_proj.weight module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight Params for bucket 9 (40901632 elements, 40901632 padded size): - module.decoder.layers.14.mlp.router.weight - module.decoder.layers.14.pre_mlp_layernorm.weight - module.decoder.layers.14.input_layernorm.weight - module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.13.self_attention.linear_kv_down_proj.weight - module.decoder.layers.13.self_attention.linear_q_proj.weight - module.decoder.layers.14.self_attention.linear_kv_down_proj.weight - module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight module.decoder.layers.14.self_attention.linear_q_proj.weight module.decoder.layers.14.self_attention.linear_proj.weight module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.13.mlp.router.weight module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.router.weight module.decoder.layers.13.self_attention.linear_kv_up_proj.weight Params for bucket 10 (41163264 elements, 41163264 padded size): module.decoder.layers.13.self_attention.linear_proj.weight @@ -810,24 +810,24 @@ Params for bucket 10 (41163264 elements, 41163264 padded size): module.decoder.layers.12.self_attention.linear_kv_up_proj.weight module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.12.self_attention.linear_kv_down_proj.weight - module.decoder.layers.12.self_attention.linear_q_proj.weight module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight module.decoder.layers.12.mlp.router.weight module.decoder.layers.12.pre_mlp_layernorm.weight - module.decoder.layers.12.input_layernorm.weight - module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.11.self_attention.linear_kv_up_proj.weight module.decoder.layers.11.self_attention.linear_kv_down_proj.weight module.decoder.layers.11.self_attention.linear_q_proj.weight module.decoder.layers.11.self_attention.linear_proj.weight - module.decoder.layers.11.pre_mlp_layernorm.weight - module.decoder.layers.11.mlp.router.weight - module.decoder.layers.11.input_layernorm.weight - module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight Params for bucket 12 (40901632 elements, 40901632 padded size): module.decoder.layers.10.mlp.router.weight @@ -836,9 +836,9 @@ Params for bucket 12 (40901632 elements, 40901632 padded size): module.decoder.layers.9.mlp.router.weight module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.self_attention.linear_q_proj.weight - module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.10.self_attention.linear_kv_down_proj.weight module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.10.self_attention.linear_q_proj.weight module.decoder.layers.10.self_attention.linear_proj.weight module.decoder.layers.10.input_layernorm.weight @@ -847,141 +847,141 @@ Params for bucket 12 (40901632 elements, 40901632 padded size): module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.9.self_attention.linear_kv_down_proj.weight Params for bucket 13 (41163264 elements, 41163264 padded size): - module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight module.decoder.layers.9.input_layernorm.weight - module.decoder.layers.8.pre_mlp_layernorm.weight - module.decoder.layers.8.input_layernorm.weight - module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight module.decoder.layers.8.self_attention.linear_kv_up_proj.weight module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.8.self_attention.linear_kv_down_proj.weight module.decoder.layers.8.self_attention.linear_q_proj.weight module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight Params for bucket 14 (42734080 elements, 42734080 padded size): module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.7.self_attention.linear_kv_down_proj.weight module.decoder.layers.7.self_attention.linear_q_proj.weight module.decoder.layers.7.self_attention.linear_proj.weight - module.decoder.layers.7.input_layernorm.weight module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.7.mlp.router.weight module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.input_layernorm.weight module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight Params for bucket 15 (40901632 elements, 40901632 padded size): - module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.6.self_attention.linear_kv_down_proj.weight - module.decoder.layers.6.self_attention.linear_kv_up_proj.weight - module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight module.decoder.layers.6.input_layernorm.weight - module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight module.decoder.layers.5.self_attention.linear_kv_up_proj.weight module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.5.self_attention.linear_kv_down_proj.weight module.decoder.layers.5.self_attention.linear_q_proj.weight - module.decoder.layers.6.mlp.router.weight - module.decoder.layers.6.pre_mlp_layernorm.weight - module.decoder.layers.6.self_attention.linear_q_proj.weight - module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight module.decoder.layers.5.mlp.router.weight module.decoder.layers.5.pre_mlp_layernorm.weight Params for bucket 16 (41163264 elements, 41163264 padded size): module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight module.decoder.layers.4.self_attention.linear_kv_up_proj.weight module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.4.self_attention.linear_kv_down_proj.weight module.decoder.layers.4.self_attention.linear_q_proj.weight module.decoder.layers.4.self_attention.linear_proj.weight - module.decoder.layers.5.input_layernorm.weight - module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.input_layernorm.weight - module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight Params for bucket 17 (42734080 elements, 42734080 padded size): - module.decoder.layers.3.pre_mlp_layernorm.weight - module.decoder.layers.3.mlp.router.weight - module.decoder.layers.3.input_layernorm.weight - module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.3.self_attention.linear_kv_up_proj.weight module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight module.decoder.layers.3.self_attention.linear_kv_down_proj.weight module.decoder.layers.3.self_attention.linear_q_proj.weight module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.3.input_layernorm.weight module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight Params for bucket 18 (40901632 elements, 40901632 padded size): - module.decoder.layers.2.pre_mlp_layernorm.weight - module.decoder.layers.2.self_attention.linear_kv_up_proj.weight module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.2.input_layernorm.weight - module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.router.weight module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.1.self_attention.linear_kv_up_proj.weight - module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.1.self_attention.linear_q_proj.weight module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.self_attention.linear_kv_down_proj.weight - module.decoder.layers.2.self_attention.linear_q_proj.weight - module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.1.self_attention.linear_kv_down_proj.weight Params for bucket 19 (71436288 elements, 71436288 padded size): module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.0.mlp.linear_fc1.weight - module.decoder.layers.1.input_layernorm.weight module.decoder.layers.0.mlp.linear_fc2.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc1.weight Params for bucket 20 (116789760 elements, 116789760 padded size): - module.decoder.layers.0.self_attention.linear_kv_up_proj.weight - module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.0.self_attention.linear_q_proj.weight module.decoder.layers.0.self_attention.linear_kv_down_proj.weight module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.0.self_attention.linear_proj.weight module.decoder.layers.0.input_layernorm.weight module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 Params for bucket 1 (40370176 elements, 40370176 padded size): - module.decoder.layers.23.mlp.experts.linear_fc2.weight6 - module.decoder.layers.23.mlp.experts.linear_fc2.weight4 - module.decoder.layers.23.mlp.experts.linear_fc2.weight1 - module.decoder.layers.23.mlp.experts.linear_fc1.weight7 - module.decoder.layers.23.mlp.experts.linear_fc1.weight5 module.decoder.layers.23.mlp.experts.linear_fc2.weight7 module.decoder.layers.23.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc2.weight3 module.decoder.layers.23.mlp.experts.linear_fc2.weight2 module.decoder.layers.23.mlp.experts.linear_fc2.weight0 module.decoder.layers.23.mlp.experts.linear_fc1.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 Params for bucket 2 (40370176 elements, 40370176 padded size): - module.decoder.layers.23.mlp.experts.linear_fc1.weight1 - module.decoder.layers.23.mlp.experts.linear_fc1.weight2 - module.decoder.layers.22.mlp.experts.linear_fc2.weight7 - module.decoder.layers.22.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc1.weight4 - module.decoder.layers.23.mlp.experts.linear_fc1.weight3 module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 module.decoder.layers.22.mlp.experts.linear_fc2.weight6 module.decoder.layers.22.mlp.experts.linear_fc2.weight4 Params for bucket 3 (40370176 elements, 40370176 padded size): module.decoder.layers.22.mlp.experts.linear_fc2.weight3 - module.decoder.layers.22.mlp.experts.linear_fc2.weight0 module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 module.decoder.layers.22.mlp.experts.linear_fc1.weight6 module.decoder.layers.22.mlp.experts.linear_fc1.weight4 module.decoder.layers.22.mlp.experts.linear_fc1.weight3 - module.decoder.layers.22.mlp.experts.linear_fc1.weight7 module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 module.decoder.layers.22.mlp.experts.linear_fc1.weight5 Params for bucket 4 (40370176 elements, 40370176 padded size): - module.decoder.layers.21.mlp.experts.linear_fc2.weight7 module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 module.decoder.layers.21.mlp.experts.linear_fc2.weight5 module.decoder.layers.21.mlp.experts.linear_fc2.weight3 module.decoder.layers.21.mlp.experts.linear_fc2.weight0 @@ -1001,139 +1001,139 @@ Params for bucket 5 (40370176 elements, 40370176 padded size): module.decoder.layers.21.mlp.experts.linear_fc1.weight1 Params for bucket 6 (40370176 elements, 40370176 padded size): module.decoder.layers.21.mlp.experts.linear_fc1.weight0 - module.decoder.layers.20.mlp.experts.linear_fc2.weight5 module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 module.decoder.layers.20.mlp.experts.linear_fc2.weight3 module.decoder.layers.20.mlp.experts.linear_fc2.weight2 module.decoder.layers.20.mlp.experts.linear_fc2.weight0 module.decoder.layers.20.mlp.experts.linear_fc1.weight6 - module.decoder.layers.20.mlp.experts.linear_fc2.weight4 module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 module.decoder.layers.20.mlp.experts.linear_fc2.weight1 module.decoder.layers.20.mlp.experts.linear_fc1.weight7 Params for bucket 7 (40370176 elements, 40370176 padded size): module.decoder.layers.20.mlp.experts.linear_fc1.weight3 module.decoder.layers.20.mlp.experts.linear_fc1.weight4 module.decoder.layers.20.mlp.experts.linear_fc1.weight0 - module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 module.decoder.layers.20.mlp.experts.linear_fc1.weight5 - module.decoder.layers.20.mlp.experts.linear_fc1.weight2 module.decoder.layers.20.mlp.experts.linear_fc1.weight1 - module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 Params for bucket 8 (40370176 elements, 40370176 padded size): - module.decoder.layers.19.mlp.experts.linear_fc2.weight5 - module.decoder.layers.19.mlp.experts.linear_fc2.weight2 - module.decoder.layers.19.mlp.experts.linear_fc2.weight3 - module.decoder.layers.19.mlp.experts.linear_fc2.weight0 - module.decoder.layers.19.mlp.experts.linear_fc1.weight6 - module.decoder.layers.19.mlp.experts.linear_fc1.weight4 module.decoder.layers.19.mlp.experts.linear_fc2.weight1 module.decoder.layers.19.mlp.experts.linear_fc2.weight4 module.decoder.layers.19.mlp.experts.linear_fc1.weight7 module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 module.decoder.layers.19.mlp.experts.linear_fc1.weight3 module.decoder.layers.19.mlp.experts.linear_fc1.weight0 module.decoder.layers.18.mlp.experts.linear_fc2.weight7 module.decoder.layers.18.mlp.experts.linear_fc2.weight5 module.decoder.layers.18.mlp.experts.linear_fc2.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight2 - module.decoder.layers.19.mlp.experts.linear_fc1.weight1 - module.decoder.layers.19.mlp.experts.linear_fc1.weight2 - module.decoder.layers.18.mlp.experts.linear_fc2.weight6 - module.decoder.layers.18.mlp.experts.linear_fc2.weight4 Params for bucket 10 (40370176 elements, 40370176 padded size): - module.decoder.layers.18.mlp.experts.linear_fc1.weight6 - module.decoder.layers.18.mlp.experts.linear_fc2.weight0 - module.decoder.layers.18.mlp.experts.linear_fc1.weight4 - module.decoder.layers.18.mlp.experts.linear_fc1.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight1 module.decoder.layers.18.mlp.experts.linear_fc1.weight7 module.decoder.layers.18.mlp.experts.linear_fc1.weight5 module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 module.decoder.layers.18.mlp.experts.linear_fc1.weight0 module.decoder.layers.17.mlp.experts.linear_fc2.weight7 module.decoder.layers.17.mlp.experts.linear_fc2.weight5 module.decoder.layers.17.mlp.experts.linear_fc2.weight3 - module.decoder.layers.17.mlp.experts.linear_fc2.weight2 module.decoder.layers.17.mlp.experts.linear_fc2.weight0 - module.decoder.layers.18.mlp.experts.linear_fc1.weight1 - module.decoder.layers.17.mlp.experts.linear_fc2.weight6 - module.decoder.layers.17.mlp.experts.linear_fc2.weight4 - module.decoder.layers.17.mlp.experts.linear_fc2.weight1 module.decoder.layers.17.mlp.experts.linear_fc1.weight7 Params for bucket 12 (40370176 elements, 40370176 padded size): module.decoder.layers.17.mlp.experts.linear_fc1.weight6 - module.decoder.layers.17.mlp.experts.linear_fc1.weight4 module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 module.decoder.layers.17.mlp.experts.linear_fc1.weight0 - module.decoder.layers.17.mlp.experts.linear_fc1.weight2 module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 module.decoder.layers.17.mlp.experts.linear_fc1.weight1 Params for bucket 13 (40370176 elements, 40370176 padded size): module.decoder.layers.16.mlp.experts.linear_fc2.weight7 - module.decoder.layers.16.mlp.experts.linear_fc2.weight5 module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 module.decoder.layers.16.mlp.experts.linear_fc2.weight2 module.decoder.layers.16.mlp.experts.linear_fc2.weight0 module.decoder.layers.16.mlp.experts.linear_fc1.weight6 - module.decoder.layers.16.mlp.experts.linear_fc2.weight4 module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 module.decoder.layers.16.mlp.experts.linear_fc2.weight1 module.decoder.layers.16.mlp.experts.linear_fc1.weight7 module.decoder.layers.16.mlp.experts.linear_fc1.weight5 Params for bucket 14 (40370176 elements, 40370176 padded size): module.decoder.layers.16.mlp.experts.linear_fc1.weight4 - module.decoder.layers.16.mlp.experts.linear_fc1.weight3 module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 module.decoder.layers.15.mlp.experts.linear_fc2.weight7 module.decoder.layers.15.mlp.experts.linear_fc2.weight5 - module.decoder.layers.16.mlp.experts.linear_fc1.weight2 module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 module.decoder.layers.15.mlp.experts.linear_fc2.weight6 module.decoder.layers.15.mlp.experts.linear_fc2.weight4 Params for bucket 15 (40370176 elements, 40370176 padded size): module.decoder.layers.15.mlp.experts.linear_fc2.weight3 - module.decoder.layers.15.mlp.experts.linear_fc2.weight2 module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 module.decoder.layers.15.mlp.experts.linear_fc1.weight6 module.decoder.layers.15.mlp.experts.linear_fc1.weight4 module.decoder.layers.15.mlp.experts.linear_fc1.weight3 - module.decoder.layers.15.mlp.experts.linear_fc2.weight1 module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 module.decoder.layers.15.mlp.experts.linear_fc1.weight5 Params for bucket 16 (40370176 elements, 40370176 padded size): module.decoder.layers.15.mlp.experts.linear_fc1.weight0 - module.decoder.layers.14.mlp.experts.linear_fc2.weight6 - module.decoder.layers.14.mlp.experts.linear_fc2.weight4 - module.decoder.layers.14.mlp.experts.linear_fc2.weight1 - module.decoder.layers.15.mlp.experts.linear_fc1.weight2 module.decoder.layers.14.mlp.experts.linear_fc2.weight7 - module.decoder.layers.15.mlp.experts.linear_fc1.weight1 module.decoder.layers.14.mlp.experts.linear_fc2.weight5 module.decoder.layers.14.mlp.experts.linear_fc2.weight3 - module.decoder.layers.14.mlp.experts.linear_fc2.weight2 module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 module.decoder.layers.14.mlp.experts.linear_fc1.weight7 module.decoder.layers.14.mlp.experts.linear_fc1.weight5 module.decoder.layers.14.mlp.experts.linear_fc1.weight2 module.decoder.layers.14.mlp.experts.linear_fc1.weight1 - module.decoder.layers.14.mlp.experts.linear_fc1.weight4 - module.decoder.layers.14.mlp.experts.linear_fc1.weight6 - module.decoder.layers.14.mlp.experts.linear_fc1.weight3 Params for bucket 18 (40370176 elements, 40370176 padded size): - module.decoder.layers.13.mlp.experts.linear_fc2.weight4 - module.decoder.layers.13.mlp.experts.linear_fc2.weight6 - module.decoder.layers.13.mlp.experts.linear_fc2.weight2 - module.decoder.layers.13.mlp.experts.linear_fc2.weight1 - module.decoder.layers.13.mlp.experts.linear_fc1.weight6 module.decoder.layers.14.mlp.experts.linear_fc1.weight0 module.decoder.layers.13.mlp.experts.linear_fc2.weight5 module.decoder.layers.13.mlp.experts.linear_fc2.weight7 module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 module.decoder.layers.13.mlp.experts.linear_fc1.weight7 Params for bucket 19 (40370176 elements, 40370176 padded size): module.decoder.layers.13.mlp.experts.linear_fc1.weight4 @@ -1141,8 +1141,8 @@ Params for bucket 19 (40370176 elements, 40370176 padded size): module.decoder.layers.13.mlp.experts.linear_fc1.weight0 module.decoder.layers.12.mlp.experts.linear_fc2.weight7 module.decoder.layers.13.mlp.experts.linear_fc1.weight5 - module.decoder.layers.13.mlp.experts.linear_fc1.weight1 module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 module.decoder.layers.12.mlp.experts.linear_fc2.weight6 Params for bucket 20 (40370176 elements, 40370176 padded size): module.decoder.layers.12.mlp.experts.linear_fc2.weight5 @@ -1157,47 +1157,47 @@ Params for bucket 20 (40370176 elements, 40370176 padded size): module.decoder.layers.12.mlp.experts.linear_fc1.weight5 Params for bucket 21 (40370176 elements, 40370176 padded size): module.decoder.layers.12.mlp.experts.linear_fc1.weight3 - module.decoder.layers.11.mlp.experts.linear_fc2.weight7 module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 module.decoder.layers.11.mlp.experts.linear_fc2.weight5 module.decoder.layers.11.mlp.experts.linear_fc2.weight3 - module.decoder.layers.11.mlp.experts.linear_fc2.weight2 - module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 module.decoder.layers.12.mlp.experts.linear_fc1.weight1 - module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 Params for bucket 22 (40370176 elements, 40370176 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight4 module.decoder.layers.11.mlp.experts.linear_fc1.weight3 - module.decoder.layers.11.mlp.experts.linear_fc1.weight5 - module.decoder.layers.11.mlp.experts.linear_fc2.weight0 - module.decoder.layers.11.mlp.experts.linear_fc1.weight7 - module.decoder.layers.11.mlp.experts.linear_fc1.weight2 Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 module.decoder.layers.10.mlp.experts.linear_fc2.weight6 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight4 module.decoder.layers.10.mlp.experts.linear_fc2.weight1 module.decoder.layers.10.mlp.experts.linear_fc1.weight7 - module.decoder.layers.11.mlp.experts.linear_fc1.weight1 module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight7 module.decoder.layers.10.mlp.experts.linear_fc2.weight3 module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight0 Params for bucket 24 (40370176 elements, 40370176 padded size): - module.decoder.layers.10.mlp.experts.linear_fc1.weight2 module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 module.decoder.layers.10.mlp.experts.linear_fc1.weight1 module.decoder.layers.10.mlp.experts.linear_fc1.weight6 - module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight0 Params for bucket 25 (40370176 elements, 40370176 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 module.decoder.layers.9.mlp.experts.linear_fc2.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc1.weight5 @@ -1208,85 +1208,85 @@ Params for bucket 25 (40370176 elements, 40370176 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc1.weight6 Params for bucket 26 (40370176 elements, 40370176 padded size): - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.8.mlp.experts.linear_fc2.weight6 - module.decoder.layers.8.mlp.experts.linear_fc2.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight3 - module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.8.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 Params for bucket 27 (40370176 elements, 40370176 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight7 - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.8.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 Params for bucket 28 (40370176 elements, 40370176 padded size): - module.decoder.layers.8.mlp.experts.linear_fc1.weight2 - module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 module.decoder.layers.7.mlp.experts.linear_fc2.weight6 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 module.decoder.layers.7.mlp.experts.linear_fc2.weight1 Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 module.decoder.layers.7.mlp.experts.linear_fc1.weight4 module.decoder.layers.7.mlp.experts.linear_fc1.weight3 - module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight2 - module.decoder.layers.7.mlp.experts.linear_fc1.weight1 Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight0 - module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 module.decoder.layers.6.mlp.experts.linear_fc2.weight2 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 module.decoder.layers.6.mlp.experts.linear_fc1.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 - module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 31 (40370176 elements, 40370176 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight3 - module.decoder.layers.6.mlp.experts.linear_fc1.weight4 - module.decoder.layers.6.mlp.experts.linear_fc1.weight0 - module.decoder.layers.5.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc1.weight5 - module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight5 - module.decoder.layers.5.mlp.experts.linear_fc2.weight3 module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 module.decoder.layers.5.mlp.experts.linear_fc2.weight0 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 module.decoder.layers.5.mlp.experts.linear_fc1.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 - module.decoder.layers.5.mlp.experts.linear_fc1.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight5 Params for bucket 33 (40370176 elements, 40370176 padded size): - module.decoder.layers.5.mlp.experts.linear_fc1.weight3 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 - module.decoder.layers.5.mlp.experts.linear_fc1.weight1 - module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 @@ -1300,64 +1300,64 @@ Params for bucket 34 (40370176 elements, 40370176 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight5 module.decoder.layers.4.mlp.experts.linear_fc1.weight2 Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 - module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight1 - module.decoder.layers.3.mlp.experts.linear_fc2.weight6 - module.decoder.layers.3.mlp.experts.linear_fc2.weight4 - module.decoder.layers.3.mlp.experts.linear_fc2.weight1 module.decoder.layers.3.mlp.experts.linear_fc1.weight7 Params for bucket 36 (40370176 elements, 40370176 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight6 - module.decoder.layers.3.mlp.experts.linear_fc1.weight4 module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 module.decoder.layers.3.mlp.experts.linear_fc1.weight0 module.decoder.layers.3.mlp.experts.linear_fc1.weight5 module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 Params for bucket 37 (40370176 elements, 40370176 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight4 - module.decoder.layers.2.mlp.experts.linear_fc2.weight6 - module.decoder.layers.2.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc1.weight7 - module.decoder.layers.2.mlp.experts.linear_fc1.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight7 - module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight3 - module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 module.decoder.layers.2.mlp.experts.linear_fc1.weight6 Params for bucket 38 (40370176 elements, 40370176 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.2.mlp.experts.linear_fc1.weight1 - module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc1.weight4 module.decoder.layers.2.mlp.experts.linear_fc1.weight0 module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight6 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight0 - module.decoder.layers.1.mlp.experts.linear_fc1.weight7 - module.decoder.layers.1.mlp.experts.linear_fc1.weight5 Params for bucket 40 (17301504 elements, 17301504 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight2 - module.decoder.layers.1.mlp.experts.linear_fc1.weight1 module.decoder.layers.1.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:48:51 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:00:57 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1365,7 +1365,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1378,14 +1378,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 16:48:51 +[after dataloaders are built] datetime: 2026-08-26 18:00:57 done with setup ... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3597.25, 3602.74) - train/valid/test-data-iterators-setup ..........: (67.08, 88.59) + model-and-optimizer-setup ......................: (3621.23, 3630.66) + train/valid/test-data-iterators-setup ..........: (70.04, 83.97) training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-26 16:48:51 +[before the start of training step] datetime: 2026-08-26 18:00:57 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1408,10 +1408,10 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0... warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -1448,14 +1448,14 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0... warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -1500,954 +1500,950 @@ WARNING:megatron.core.rerun_state_machine:Result validation enabled /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49060 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49060 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=61440 expected_recv=48835 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49208 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49416 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49163 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49061 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49163 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49404 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49060 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=61440 expected_recv=48835 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49208 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49416 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49061 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49163 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49061 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49250 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49404 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49250 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49250 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49050 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49046 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49138 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49050 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49194 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49046 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49050 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49619 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48991 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49194 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49619 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49050 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48991 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49303 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49619 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49278 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48913 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49619 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49303 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49124 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49278 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48913 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48913 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49097 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49134 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49124 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48761 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48913 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49097 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49134 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48761 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49124 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=61440 expected_recv=48753 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49124 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49071 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=61440 expected_recv=48753 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49071 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=61440 expected_recv=48753 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49552 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49159 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49254 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=61440 expected_recv=48753 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49552 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49254 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49159 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49153 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49153 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49332 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49335 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49416 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49012 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49028 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49332 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49335 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49332 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49229 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49281 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49245 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49012 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49028 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49332 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49229 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49281 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49245 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49404 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49416 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48767 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49315 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=61440 expected_recv=48835 expected_overflow=0 exact_recv_per_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49335 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48767 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49145 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49397 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49315 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=61440 expected_recv=49404 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49335 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49145 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49397 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=61440 expected_recv=48835 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49351 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48991 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49046 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49485 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49046 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49151 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48944 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49351 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49485 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=61440 expected_recv=48801 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49151 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49059 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48944 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=61440 expected_recv=48991 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=61440 expected_recv=48801 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49157 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49351 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49059 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49157 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49334 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49334 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49359 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49351 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49303 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49281 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49303 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49281 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49348 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49359 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49012 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48944 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48800 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49348 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=61440 expected_recv=49138 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48944 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48981 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49012 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48761 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48800 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49359 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48981 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49232 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49359 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=61440 expected_recv=48761 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49215 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49097 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49336 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49215 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49232 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48957 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49336 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49097 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49012 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48957 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48811 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49278 expected_overflow=0 exact_recv_per_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49305 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49012 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49348 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49305 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48811 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49348 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49336 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49153 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49087 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=61440 expected_recv=48811 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=61440 expected_recv=49278 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49336 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49153 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49087 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49254 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49254 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49087 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49552 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49087 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48888 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49398 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49552 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49142 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48888 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49142 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48625 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49398 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49179 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48625 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49281 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=61440 expected_recv=48707 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49179 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49228 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49482 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49281 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=61440 expected_recv=48707 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49228 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49398 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49030 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49482 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49229 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49398 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49030 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49367 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49229 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49367 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49112 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49012 expected_overflow=0 exact_recv_per_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49228 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49112 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=61440 expected_recv=49012 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49397 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49228 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49117 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49121 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49397 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49117 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=61440 expected_recv=48995 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49121 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49367 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49145 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=61440 expected_recv=48873 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=61440 expected_recv=49145 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=61440 expected_recv=48873 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=61440 expected_recv=48995 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49083 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49334 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49283 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49367 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48767 expected_overflow=0 exact_recv_per_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49117 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49083 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49334 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49283 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=61440 expected_recv=48767 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49157 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49117 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49015 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49316 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49157 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49015 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49316 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=61440 expected_recv=48899 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=61440 expected_recv=48801 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49425 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=61440 expected_recv=48801 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49425 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=61440 expected_recv=48899 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=61440 expected_recv=48873 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49398 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=12 calls=1 budget=61440 expected_recv=48873 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49215 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49398 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49413 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49485 expected_overflow=0 exact_recv_per_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49413 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48981 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49215 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=61440 expected_recv=49485 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48981 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49425 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49206 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49425 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49501 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49206 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49075 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49501 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=61440 expected_recv=48992 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49075 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=61440 expected_recv=48992 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48800 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49103 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=61440 expected_recv=48800 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49396 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49103 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49605 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49075 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49075 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=61440 expected_recv=48992 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49396 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=61440 expected_recv=48707 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=61440 expected_recv=48916 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49182 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49605 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49348 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49281 expected_overflow=0 exact_recv_per_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=14 calls=1 budget=61440 expected_recv=48992 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=10 calls=1 budget=61440 expected_recv=48707 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=61440 expected_recv=48916 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49182 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49348 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=61440 expected_recv=49281 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49126 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49025 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49305 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49126 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=61440 expected_recv=48764 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49025 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49305 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=61440 expected_recv=48916 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48505 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=61440 expected_recv=48764 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49281 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48505 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49479 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49182 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=14 calls=1 budget=61440 expected_recv=48916 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49182 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49142 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49281 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49479 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49182 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49182 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49142 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49232 expected_overflow=0 exact_recv_per_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49517 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=61440 expected_recv=49232 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49224 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48888 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49517 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49390 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49224 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49281 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=61440 expected_recv=48888 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49390 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=61440 expected_recv=48812 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49316 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48854 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=61440 expected_recv=48812 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49281 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49479 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49316 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48854 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49479 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=61440 expected_recv=48997 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49087 expected_overflow=0 exact_recv_per_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49244 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=61440 expected_recv=49087 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=61440 expected_recv=48997 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49317 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49244 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49068 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49317 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49030 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48854 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49068 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49341 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49179 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48971 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49426 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49030 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48854 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49341 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49179 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48971 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49426 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49083 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49179 expected_overflow=0 exact_recv_per_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49142 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49083 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49142 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48895 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=10 calls=1 budget=61440 expected_recv=49179 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48895 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=61440 expected_recv=48995 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49179 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=11 calls=1 budget=61440 expected_recv=48995 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49329 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48982 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49430 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49179 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48971 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49329 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48982 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49430 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48971 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49398 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49050 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49229 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49398 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49050 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49375 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=11 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49229 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49375 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=61440 expected_recv=48899 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49138 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48982 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=12 calls=1 budget=61440 expected_recv=48899 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49585 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49585 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49477 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=18 calls=1 budget=61440 expected_recv=48982 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48876 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49477 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49206 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48876 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48907 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49283 expected_overflow=0 exact_recv_per_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48790 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49206 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48907 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=12 calls=1 budget=61440 expected_recv=49283 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48790 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49145 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49145 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49244 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48876 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49192 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49244 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48896 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48876 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48819 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49192 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49309 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48896 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48819 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49309 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49396 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49414 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=61440 expected_recv=48902 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49058 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49414 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49396 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=13 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=61440 expected_recv=48902 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49058 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49060 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49293 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49148 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48819 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49032 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49060 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49293 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49133 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49148 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48819 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49192 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49032 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49133 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49192 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49191 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=61440 expected_recv=48764 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49501 expected_overflow=0 exact_recv_per_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49191 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=15 calls=1 budget=61440 expected_recv=48764 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=14 calls=1 budget=61440 expected_recv=49501 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49126 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49229 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49126 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49032 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49229 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49294 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49293 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49032 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49294 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49293 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48890 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49390 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48890 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49605 expected_overflow=0 exact_recv_per_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49390 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=15 calls=1 budget=61440 expected_recv=49605 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49517 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=16 calls=1 budget=61440 expected_recv=49517 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49294 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48790 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=19 calls=1 budget=61440 expected_recv=48790 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49294 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49068 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48505 expected_overflow=0 exact_recv_per_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=17 calls=1 budget=61440 expected_recv=49068 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=61440 expected_recv=48997 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=16 calls=1 budget=61440 expected_recv=48505 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=17 calls=1 budget=61440 expected_recv=48997 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=61440 expected_recv=48812 expected_overflow=0 exact_recv_per_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=17 calls=1 budget=61440 expected_recv=48812 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49414 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49142 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49142 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49414 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49191 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49191 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48890 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49341 expected_overflow=0 exact_recv_per_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49138 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=18 calls=1 budget=61440 expected_recv=49341 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=22 calls=1 budget=61440 expected_recv=48890 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49050 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49050 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49145 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49329 expected_overflow=0 exact_recv_per_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49145 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=19 calls=1 budget=61440 expected_recv=49329 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48907 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=20 calls=1 budget=61440 expected_recv=48907 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=61440 expected_recv=48902 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=21 calls=1 budget=61440 expected_recv=48902 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49585 expected_overflow=0 exact_recv_per_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=20 calls=1 budget=61440 expected_recv=49585 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49309 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49309 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=21 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49133 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49133 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49060 expected_overflow=0 exact_recv_per_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=22 calls=1 budget=61440 expected_recv=49060 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[49060, 49163, 49069, 49404, 49061, 49416, 49208, 48835] avg=49152.0000 max=49416 max_rank=5 min=48835 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[49250, 49194, 48991, 49296, 49050, 49046, 49251, 49138] avg=49152.0000 max=49296 max_rank=3 min=48991 min_rank=2 imbalance_ratio=1.002930 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[49619, 49134, 48761, 49097, 48913, 49303, 49111, 49278] avg=49152.0000 max=49619 max_rank=0 min=48761 min_rank=2 imbalance_ratio=1.009501 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[49124, 49159, 49254, 49552, 48753, 49237, 49071, 49066] avg=49152.0000 max=49552 max_rank=3 min=48753 min_rank=4 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[48936, 49245, 49281, 49229, 49028, 49153, 49332, 49012] avg=49152.0000 max=49332 max_rank=6 min=48936 min_rank=0 imbalance_ratio=1.003662 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[49335, 48927, 49397, 49145, 49315, 49205, 49125, 48767] avg=49152.0000 max=49397 max_rank=2 min=48767 min_rank=7 imbalance_ratio=1.004985 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[49022, 49059, 49157, 48801, 49151, 49190, 49351, 49485] avg=49152.0000 max=49485 max_rank=7 min=48801 min_rank=3 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[48944, 49169, 48981, 48800, 49348, 49334, 49359, 49281] avg=49152.0000 max=49359 max_rank=6 min=48800 min_rank=3 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[49012, 48811, 49348, 49305, 48957, 49215, 49336, 49232] avg=49152.0000 max=49348 max_rank=2 min=48811 min_rank=1 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[49649, 48611, 49142, 48888, 49087, 49099, 49653, 49087] avg=49152.0000 max=49653 max_rank=6 min=48611 min_rank=1 imbalance_ratio=1.010193 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=11 tokens_per_ep_rank=[49644, 48625, 49142, 48888, 49087, 49099, 49644, 49087] avg=49152.0000 max=49644 max_rank=0 min=48625 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=9 max_load_reduction_pct=0.000181 moved_to_replicas=14 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=12 tokens_per_ep_rank=[49398, 49112, 49080, 49030, 49482, 48707, 49228, 49179] avg=49152.0000 max=49482 max_rank=4 min=48707 min_rank=5 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=13 tokens_per_ep_rank=[49367, 49051, 49083, 48995, 49121, 49260, 49117, 49222] avg=49152.0000 max=49367 max_rank=0 min=48995 min_rank=3 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=14 tokens_per_ep_rank=[48873, 49413, 49398, 48899, 49015, 49316, 49019, 49283] avg=49152.0000 max=49413 max_rank=1 min=48873 min_rank=0 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=15 tokens_per_ep_rank=[49425, 49075, 49206, 49237, 49169, 49044, 49057, 49003] avg=49152.0000 max=49425 max_rank=0 min=49003 min_rank=7 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=16 tokens_per_ep_rank=[49103, 48916, 49396, 49044, 48992, 49231, 49033, 49501] avg=49152.0000 max=49501 max_rank=7 min=48916 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=17 tokens_per_ep_rank=[49025, 49281, 48764, 49126, 49182, 49051, 49182, 49605] avg=49152.0000 max=49605 max_rank=7 min=48764 min_rank=2 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=18 tokens_per_ep_rank=[49224, 48854, 49390, 49517, 49479, 49244, 49003, 48505] avg=49152.0000 max=49517 max_rank=3 min=48505 min_rank=7 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=19 tokens_per_ep_rank=[49317, 49179, 49068, 48997, 49125, 49292, 49426, 48812] avg=49152.0000 max=49426 max_rank=6 min=48812 min_rank=7 imbalance_ratio=1.005575 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=20 tokens_per_ep_rank=[48895, 48982, 49226, 49142, 48971, 49229, 49430, 49341] avg=49152.0000 max=49430 max_rank=6 min=48895 min_rank=0 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=23 total_expected_recv=1130990 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269184 peak_demand=49605 required_capacity=1.009216 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=23 total_expected_recv=1130161 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271616 peak_demand=49482 required_capacity=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=21 tokens_per_ep_rank=[49375, 48876, 49138, 49050, 49181, 48790, 49477, 49329] avg=49152.0000 max=49477 max_rank=6 min=48790 min_rank=5 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=23 total_expected_recv=1129903 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270976 peak_demand=49552 required_capacity=1.008138 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=23 total_expected_recv=1131337 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49644 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=23 total_expected_recv=1130414 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270336 peak_demand=49398 required_capacity=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=23 total_expected_recv=1128370 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272512 peak_demand=49413 required_capacity=1.005310 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=23 total_expected_recv=1131936 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269696 peak_demand=49644 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=23 total_expected_recv=1130857 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271104 peak_demand=49416 required_capacity=1.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=22 tokens_per_ep_rank=[49154, 48819, 49145, 48907, 49296, 49414, 48896, 49585] avg=49152.0000 max=49585 max_rank=7 min=48819 min_rank=1 imbalance_ratio=1.008809 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=23 tokens_per_ep_rank=[49058, 49032, 48902, 49309, 49192, 49191, 49148, 49384] avg=49152.0000 max=49384 max_rank=7 min=48902 min_rank=2 imbalance_ratio=1.004720 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=23 total_expected_recv=1128370 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272512 peak_demand=49413 required_capacity=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=24 tokens_per_ep_rank=[49177, 49294, 49237, 49133, 49293, 48890, 49132, 49060] avg=49152.0000 max=49294 max_rank=1 min=48890 min_rank=5 imbalance_ratio=1.002889 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=23 total_expected_recv=1130414 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270336 peak_demand=49398 required_capacity=1.005005 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=23 total_expected_recv=1129903 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270976 peak_demand=49552 required_capacity=1.008138 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=23 total_expected_recv=1130990 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269184 peak_demand=49605 required_capacity=1.009216 + [2026-08-26 18:01:25] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 27444.7 | throughput per GPU (TFLOP/s/GPU): 4.6 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092169E+01 | loss scale: 1.0 | grad norm: 30.280 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | Number of parameters in transformer block in billions: 13.53 - [2026-08-26 16:49:19] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 27765.9 | throughput per GPU (TFLOP/s/GPU): 4.6 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092169E+01 | loss scale: 1.0 | grad norm: 30.280 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 |Number of parameters in embedding layers in billions: 0.21 - +Number of parameters in embedding layers in billions: 0.21 Total number of parameters in billions: 13.74 Number of parameters in most loaded shard in billions: 13.7356 Theoretical memory footprints: weight and optimizer=98244.96 MB [Rank 0] (after 1 iterations) memory (MB) | allocated: 37109.60302734375 | max allocated: 104508.90771484375 | reserved: 107726.0 | max reserved: 107726.0 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -2456,6 +2452,10 @@ Theoretical memory footprints: weight and optimizer=98244.96 MB warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -2483,10242 +2483,10240 @@ Theoretical memory footprints: weight and optimizer=98244.96 MB /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49334 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49095 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49150 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49082 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=61440 expected_recv=48980 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49095 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49302 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49082 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49095 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49334 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49150 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49082 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49216 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=61440 expected_recv=48980 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49095 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49302 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49302 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49082 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49216 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49289 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49284 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49216 expected_overflow=0 exact_recv_per_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49309 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49289 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49284 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49289 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49395 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49284 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49309 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=61440 expected_recv=48839 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49395 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=61440 expected_recv=48791 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49284 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=61440 expected_recv=48839 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49289 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49395 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=61440 expected_recv=48791 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49309 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=61440 expected_recv=49216 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49360 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=61440 expected_recv=49309 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49081 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49360 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49081 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49092 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48996 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49504 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48931 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49360 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49092 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49504 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49081 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49048 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48931 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49092 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48996 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49048 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49092 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48996 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49504 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=61440 expected_recv=48839 expected_overflow=0 exact_recv_per_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48931 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48684 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49247 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49361 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=61440 expected_recv=48931 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=61440 expected_recv=48839 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49361 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48962 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48684 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48998 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49247 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48684 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49170 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48962 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49342 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49361 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48998 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49452 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49247 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48684 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49170 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49452 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49342 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49361 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49452 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49247 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48998 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49048 expected_overflow=0 exact_recv_per_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=61440 expected_recv=48998 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=61440 expected_recv=49048 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49272 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49272 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48853 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48982 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49376 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49272 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48853 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48982 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49376 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49272 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48815 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48982 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49456 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49435 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48815 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49342 expected_overflow=0 exact_recv_per_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49435 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48815 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=61440 expected_recv=48982 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49456 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49198 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=61440 expected_recv=49342 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49198 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49527 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49198 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49327 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49527 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49198 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49527 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49327 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48949 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49527 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48949 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49245 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48855 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48679 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49327 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49435 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49456 expected_overflow=0 exact_recv_per_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49245 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49139 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48855 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48949 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49327 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48679 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=61440 expected_recv=49435 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=61440 expected_recv=49456 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49139 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48949 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49550 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49139 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49178 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49550 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49139 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=61440 expected_recv=48749 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49178 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=61440 expected_recv=48983 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=61440 expected_recv=48983 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48813 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49245 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=61440 expected_recv=48749 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48679 expected_overflow=0 exact_recv_per_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48813 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49162 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=61440 expected_recv=48749 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=61440 expected_recv=48983 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49161 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=61440 expected_recv=48679 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49162 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49245 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49178 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=61440 expected_recv=48983 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49178 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48988 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49162 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49299 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49568 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49162 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49311 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49299 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49135 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49568 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48940 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49311 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48813 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49135 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48940 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48813 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49299 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49135 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49299 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49014 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49311 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49175 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49014 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=61440 expected_recv=49311 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=61440 expected_recv=49161 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49014 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49175 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49175 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49100 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49411 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49014 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49030 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49411 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49100 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49030 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49175 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49100 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49030 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=61440 expected_recv=48958 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=61440 expected_recv=48958 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48940 expected_overflow=0 exact_recv_per_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=61440 expected_recv=48940 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49119 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=61440 expected_recv=48920 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49184 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49119 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=61440 expected_recv=48920 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=61440 expected_recv=48958 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49447 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49184 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=61440 expected_recv=48958 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49447 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49184 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49085 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49085 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49074 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49411 expected_overflow=0 exact_recv_per_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49074 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=61440 expected_recv=49411 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49095 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49085 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49095 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49095 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49152 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49085 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49095 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48908 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48908 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49447 expected_overflow=0 exact_recv_per_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49152 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49425 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49152 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48882 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49308 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=61440 expected_recv=49447 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48908 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49425 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49293 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48882 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49308 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49293 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49334 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49308 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49349 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48908 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49349 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49334 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48882 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49176 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49176 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=11 calls=1 budget=61440 expected_recv=48882 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49176 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49176 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48839 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49334 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48973 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48839 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49272 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49334 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48973 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=10 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49097 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49228 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49176 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49349 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49272 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49097 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49228 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=12 calls=1 budget=61440 expected_recv=49349 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48839 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49416 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49293 expected_overflow=0 exact_recv_per_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48960 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48839 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49416 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48960 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49493 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48892 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49493 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48892 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=11 calls=1 budget=61440 expected_recv=49293 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48960 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49158 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48892 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48892 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49228 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49092 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49158 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49228 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49092 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49493 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49220 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48973 expected_overflow=0 exact_recv_per_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=13 calls=1 budget=61440 expected_recv=49493 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49220 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=61440 expected_recv=48834 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49092 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=61440 expected_recv=48897 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=61440 expected_recv=48834 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49092 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=61440 expected_recv=48897 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49220 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=12 calls=1 budget=61440 expected_recv=48973 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49477 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49220 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49584 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49232 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49477 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49584 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49232 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=14 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=61440 expected_recv=48834 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49098 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48892 expected_overflow=0 exact_recv_per_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=14 calls=1 budget=61440 expected_recv=48834 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49098 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48869 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48948 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48869 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48948 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=13 calls=1 budget=61440 expected_recv=48892 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49025 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49232 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49042 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49025 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49232 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49042 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49458 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49584 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49134 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49458 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=15 calls=1 budget=61440 expected_recv=49584 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49134 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=61440 expected_recv=48717 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49185 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49144 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=61440 expected_recv=48717 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48869 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=61440 expected_recv=48897 expected_overflow=0 exact_recv_per_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49185 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49144 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48869 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49134 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=14 calls=1 budget=61440 expected_recv=48897 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49508 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49458 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48930 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49134 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49028 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49508 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49458 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48930 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49028 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49042 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49396 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49042 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49339 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48817 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49396 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=61440 expected_recv=48717 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48948 expected_overflow=0 exact_recv_per_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49339 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48817 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49046 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=16 calls=1 budget=61440 expected_recv=48717 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=15 calls=1 budget=61440 expected_recv=48948 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49046 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49243 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49046 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49151 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49151 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48930 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49243 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48974 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48930 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48824 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48974 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49223 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49396 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49144 expected_overflow=0 exact_recv_per_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48824 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49174 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49223 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49412 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=17 calls=1 budget=61440 expected_recv=49396 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49151 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49243 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49174 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49412 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=16 calls=1 budget=61440 expected_recv=49144 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49412 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49151 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49273 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49243 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49412 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49273 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49322 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49073 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49073 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=61440 expected_recv=48899 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49223 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48974 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=19 calls=1 budget=61440 expected_recv=48899 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49223 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49178 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49094 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48817 expected_overflow=0 exact_recv_per_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=18 calls=1 budget=61440 expected_recv=48974 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49094 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=17 calls=1 budget=61440 expected_recv=48817 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49322 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49178 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=61440 expected_recv=48703 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49094 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49174 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=61440 expected_recv=48703 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49306 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49174 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49306 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49015 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49073 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49015 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=61440 expected_recv=48703 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49223 expected_overflow=0 exact_recv_per_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49073 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49293 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49202 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=20 calls=1 budget=61440 expected_recv=48703 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=18 calls=1 budget=61440 expected_recv=49223 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49193 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49293 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49202 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49193 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49178 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49178 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49138 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49255 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49306 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49255 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=61440 expected_recv=48729 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49040 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49202 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=61440 expected_recv=48729 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49040 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49306 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49223 expected_overflow=0 exact_recv_per_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49202 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49352 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49193 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=19 calls=1 budget=61440 expected_recv=49223 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49352 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49193 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=61440 expected_recv=48686 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49133 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=61440 expected_recv=48686 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49133 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49235 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49235 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=21 calls=1 budget=61440 expected_recv=49138 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49110 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=20 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49110 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49040 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49040 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=61440 expected_recv=48686 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=61440 expected_recv=48729 expected_overflow=0 exact_recv_per_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=22 calls=1 budget=61440 expected_recv=48686 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=21 calls=1 budget=61440 expected_recv=48729 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49235 expected_overflow=0 exact_recv_per_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=22 calls=1 budget=61440 expected_recv=49235 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[49216, 49302, 49057, 48980, 49082, 49150, 49334, 49095] avg=49152.0000 max=49334 max_rank=6 min=48980 min_rank=3 imbalance_ratio=1.003703 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[48839, 48791, 49309, 49395, 49105, 49204, 49289, 49284] avg=49152.0000 max=49395 max_rank=3 min=48791 min_rank=1 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[49048, 48996, 48931, 49092, 49504, 49081, 49360, 49204] avg=49152.0000 max=49504 max_rank=4 min=48931 min_rank=2 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[49342, 49452, 48998, 49170, 49247, 48962, 48684, 49361] avg=49152.0000 max=49452 max_rank=1 min=48684 min_rank=6 imbalance_ratio=1.006104 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[49456, 49211, 48815, 48982, 49376, 49272, 48853, 49251] avg=49152.0000 max=49456 max_rank=0 min=48815 min_rank=2 imbalance_ratio=1.006185 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[48679, 48949, 48855, 49327, 49527, 49198, 49246, 49435] avg=49152.0000 max=49527 max_rank=4 min=48679 min_rank=0 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[49161, 48983, 48749, 49178, 49211, 49139, 49550, 49245] avg=49152.0000 max=49550 max_rank=6 min=48749 min_rank=2 imbalance_ratio=1.008097 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[48940, 49135, 49311, 49299, 49568, 49162, 48988, 48813] avg=49152.0000 max=49568 max_rank=4 min=48813 min_rank=7 imbalance_ratio=1.008464 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[49411, 49030, 49222, 49100, 49175, 49231, 49014, 49033] avg=49152.0000 max=49411 max_rank=0 min=49014 min_rank=6 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=11 tokens_per_ep_rank=[49447, 49246, 49251, 49184, 48920, 49091, 49119, 48958] avg=49152.0000 max=49447 max_rank=0 min=48920 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=12 tokens_per_ep_rank=[49132, 49164, 49152, 49095, 49137, 49074, 49377, 49085] avg=49152.0000 max=49377 max_rank=6 min=49074 min_rank=5 imbalance_ratio=1.004578 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=13 tokens_per_ep_rank=[49293, 49137, 48882, 49308, 49425, 49152, 48908, 49111] avg=49152.0000 max=49425 max_rank=4 min=48882 min_rank=2 imbalance_ratio=1.005554 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=14 tokens_per_ep_rank=[48973, 49097, 48839, 49272, 49176, 49176, 49349, 49334] avg=49152.0000 max=49349 max_rank=6 min=48839 min_rank=2 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=15 tokens_per_ep_rank=[48892, 49158, 49493, 48892, 48960, 49416, 49228, 49177] avg=49152.0000 max=49493 max_rank=2 min=48892 min_rank=0 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=16 tokens_per_ep_rank=[48897, 49211, 48834, 49477, 49220, 49132, 49353, 49092] avg=49152.0000 max=49477 max_rank=3 min=48834 min_rank=2 imbalance_ratio=1.006612 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=17 tokens_per_ep_rank=[48948, 49260, 48869, 49025, 49200, 49098, 49584, 49232] avg=49152.0000 max=49584 max_rank=6 min=48869 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=18 tokens_per_ep_rank=[49144, 49028, 48717, 49508, 49134, 49185, 49042, 49458] avg=49152.0000 max=49508 max_rank=3 min=48717 min_rank=2 imbalance_ratio=1.007243 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=19 tokens_per_ep_rank=[48817, 49237, 49396, 49046, 49237, 49339, 48930, 49214] avg=49152.0000 max=49396 max_rank=2 min=48817 min_rank=0 imbalance_ratio=1.004964 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=20 tokens_per_ep_rank=[49223, 49273, 48974, 49412, 49151, 48824, 49116, 49243] avg=49152.0000 max=49412 max_rank=3 min=48824 min_rank=5 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=21 tokens_per_ep_rank=[49223, 49258, 49073, 49094, 49322, 48899, 49174, 49173] avg=49152.0000 max=49322 max_rank=4 min=48899 min_rank=5 imbalance_ratio=1.003459 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=22 tokens_per_ep_rank=[49227, 49171, 49306, 49293, 48703, 49015, 49178, 49323] avg=49152.0000 max=49323 max_rank=7 min=48703 min_rank=4 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=23 tokens_per_ep_rank=[48729, 49096, 49138, 49352, 49202, 49255, 49251, 49193] avg=49152.0000 max=49352 max_rank=3 min=48729 min_rank=0 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=24 tokens_per_ep_rank=[49235, 49110, 48686, 49401, 49258, 49133, 49040, 49353] avg=49152.0000 max=49401 max_rank=3 min=48686 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=23 total_expected_recv=1130188 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49416 required_capacity=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=23 total_expected_recv=1131667 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269696 peak_demand=49458 required_capacity=1.006226 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=23 total_expected_recv=1131840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269056 peak_demand=49568 required_capacity=1.008464 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=23 total_expected_recv=1131882 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269312 peak_demand=49508 required_capacity=1.007243 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=23 total_expected_recv=1130295 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49452 required_capacity=1.006104 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=23 total_expected_recv=1130967 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49584 required_capacity=1.008789 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=23 total_expected_recv=1127857 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273792 peak_demand=49493 required_capacity=1.006938 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=23 total_expected_recv=1130188 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49416 required_capacity=1.005371 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=23 total_expected_recv=1129272 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272000 peak_demand=49456 required_capacity=1.006185 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=23 total_expected_recv=1130295 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49452 required_capacity=1.006104 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=23 total_expected_recv=1131882 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269312 peak_demand=49508 required_capacity=1.007243 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=23 total_expected_recv=1131667 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269696 peak_demand=49458 required_capacity=1.006226 - [2026-08-26 16:49:19] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 636.5 | throughput per GPU (TFLOP/s/GPU): 200.4 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091868E+01 | loss scale: 1.0 | grad norm: 30.184 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49464 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49532 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] + [2026-08-26 18:01:26] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 641.3 | throughput per GPU (TFLOP/s/GPU): 198.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091868E+01 | loss scale: 1.0 | grad norm: 30.184 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49122 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48705 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48660 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49464 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49532 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49464 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49122 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48705 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48660 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49532 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49460 expected_overflow=0 exact_recv_per_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49532 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49122 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48705 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49464 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=61440 expected_recv=48660 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=61440 expected_recv=49460 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49344 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49189 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49417 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49344 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=61440 expected_recv=48863 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49012 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49344 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49417 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=61440 expected_recv=48863 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49189 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49347 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49012 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49417 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49344 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=61440 expected_recv=48863 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49012 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49347 expected_overflow=0 exact_recv_per_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=61440 expected_recv=49347 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49249 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48925 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48925 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49172 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49249 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49163 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49152 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49172 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49249 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49172 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49152 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49163 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49419 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48925 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49172 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=61440 expected_recv=48925 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49419 expected_overflow=0 exact_recv_per_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=61440 expected_recv=49419 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49216 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48704 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48704 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49385 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49148 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49385 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49216 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49088 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49148 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49216 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49133 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49088 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49385 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49148 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48704 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49385 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49148 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=61440 expected_recv=48704 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49133 expected_overflow=0 exact_recv_per_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=61440 expected_recv=48586 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=61440 expected_recv=49133 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=61440 expected_recv=48586 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49535 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49554 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49535 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49092 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49554 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=61440 expected_recv=48771 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49092 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49092 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=61440 expected_recv=48586 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49535 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49334 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49092 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=61440 expected_recv=48771 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=61440 expected_recv=48586 actual_valid_recv=49024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49535 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=61440 expected_recv=48771 expected_overflow=0 exact_recv_per_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=61440 expected_recv=49334 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49601 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=61440 expected_recv=48771 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49463 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49601 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49463 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48972 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49310 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48972 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48983 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48983 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49152 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49601 expected_overflow=0 exact_recv_per_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49463 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48972 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49310 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=61440 expected_recv=48983 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=61440 expected_recv=49601 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49213 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49213 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49335 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49092 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49039 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49335 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49092 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=61440 expected_recv=48873 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49039 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49335 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=61440 expected_recv=48873 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49310 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49335 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49213 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49310 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=61440 expected_recv=48873 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49092 expected_overflow=0 exact_recv_per_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49213 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=61440 expected_recv=48873 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=61440 expected_recv=49092 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49106 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=61440 expected_recv=48949 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=61440 expected_recv=48949 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49106 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49577 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=61440 expected_recv=48873 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49106 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49065 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49039 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49577 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=61440 expected_recv=48873 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49577 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49039 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49106 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=61440 expected_recv=48949 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=61440 expected_recv=48753 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49065 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49577 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=61440 expected_recv=48949 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=61440 expected_recv=48753 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49366 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49343 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49366 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49343 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49170 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=61440 expected_recv=48837 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49170 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49065 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49343 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49439 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49170 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=61440 expected_recv=48837 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49065 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49343 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49439 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49366 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49201 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49170 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49055 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49527 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=61440 expected_recv=48837 expected_overflow=0 exact_recv_per_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49366 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=61440 expected_recv=49201 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49527 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=61440 expected_recv=48837 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49055 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=61440 expected_recv=48811 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49385 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49439 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=61440 expected_recv=48811 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49385 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=61440 expected_recv=48811 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49527 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49439 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49102 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=61440 expected_recv=48811 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=61440 expected_recv=48811 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49527 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49055 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=61440 expected_recv=48811 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49055 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49265 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49236 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49265 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49065 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49102 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49236 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49065 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49102 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=61440 expected_recv=48811 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49265 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48941 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49265 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49065 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49065 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48941 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49265 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49236 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48983 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49265 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49236 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49442 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49341 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48983 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49442 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49341 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48983 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=10 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48792 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49504 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=61440 expected_recv=48977 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48792 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49504 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49341 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49189 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49341 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=61440 expected_recv=48977 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49189 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49062 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49139 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49062 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=61440 expected_recv=48901 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49442 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48941 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48792 expected_overflow=0 exact_recv_per_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49139 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=12 calls=1 budget=61440 expected_recv=48901 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49442 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49420 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49131 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=11 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48941 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=11 calls=1 budget=61440 expected_recv=48792 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49139 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49131 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49420 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49185 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49139 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49131 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49216 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49185 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49216 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49282 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49184 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49141 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49062 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=61440 expected_recv=48977 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49282 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49184 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49141 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49420 expected_overflow=0 exact_recv_per_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49062 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=61440 expected_recv=48797 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=12 calls=1 budget=61440 expected_recv=48977 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48903 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49420 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49184 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=12 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49141 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=13 calls=1 budget=61440 expected_recv=48797 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49184 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49141 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48903 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49141 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48968 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49141 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49120 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48968 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49526 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49185 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49282 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49120 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49526 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49185 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49282 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49216 expected_overflow=0 exact_recv_per_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49084 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49141 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49526 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49084 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49141 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=13 calls=1 budget=61440 expected_recv=49216 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49526 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49241 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49120 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49600 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49241 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49120 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48903 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49141 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=61440 expected_recv=48921 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49416 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48968 expected_overflow=0 exact_recv_per_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48903 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=14 calls=1 budget=61440 expected_recv=49141 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49416 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49600 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=61440 expected_recv=48921 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49058 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=14 calls=1 budget=61440 expected_recv=48968 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49416 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49058 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49058 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49058 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=15 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49084 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=61440 expected_recv=48921 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=61440 expected_recv=48939 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49054 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49084 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=15 calls=1 budget=61440 expected_recv=48921 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=61440 expected_recv=48939 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49054 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49087 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=61440 expected_recv=48939 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49087 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=15 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=16 calls=1 budget=61440 expected_recv=48939 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49243 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49243 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49371 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49054 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49371 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49218 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49218 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49229 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49054 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49600 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49087 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49120 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=61440 expected_recv=48996 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49600 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49087 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49058 expected_overflow=0 exact_recv_per_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49229 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49120 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=17 calls=1 budget=61440 expected_recv=48996 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49218 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=16 calls=1 budget=61440 expected_recv=49058 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=61440 expected_recv=48977 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49218 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49371 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=61440 expected_recv=48977 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49371 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49113 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49394 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49120 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=61440 expected_recv=48801 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49243 expected_overflow=0 exact_recv_per_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49120 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49113 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49394 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=18 calls=1 budget=61440 expected_recv=48801 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49348 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49113 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=61440 expected_recv=48754 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=17 calls=1 budget=61440 expected_recv=49243 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49332 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49027 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49332 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49027 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49348 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49348 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49113 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=61440 expected_recv=48754 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49229 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49394 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49431 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49229 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=18 calls=1 budget=61440 expected_recv=49394 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49155 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49235 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=61440 expected_recv=48977 expected_overflow=0 exact_recv_per_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49431 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=61440 expected_recv=48884 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49186 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=61440 expected_recv=48754 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49431 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49155 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49235 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=19 calls=1 budget=61440 expected_recv=48754 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49308 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49144 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49431 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49308 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49144 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=18 calls=1 budget=61440 expected_recv=48977 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=61440 expected_recv=48884 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49186 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49186 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49027 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49027 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49155 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49088 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49155 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=61440 expected_recv=48884 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49400 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=61440 expected_recv=48930 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49088 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=61440 expected_recv=48870 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49332 expected_overflow=0 exact_recv_per_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=20 calls=1 budget=61440 expected_recv=48884 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49400 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49088 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=61440 expected_recv=48870 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49088 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49071 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=61440 expected_recv=48930 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=20 calls=1 budget=61440 expected_recv=48870 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=19 calls=1 budget=61440 expected_recv=49332 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49144 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49071 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=61440 expected_recv=48930 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49144 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49472 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49017 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49472 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49380 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=61440 expected_recv=48797 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=21 calls=1 budget=61440 expected_recv=48930 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49017 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=61440 expected_recv=48797 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49427 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49472 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49427 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=61440 expected_recv=48926 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49308 expected_overflow=0 exact_recv_per_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49017 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49472 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49327 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49380 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=21 calls=1 budget=61440 expected_recv=48926 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=20 calls=1 budget=61440 expected_recv=49308 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49380 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49327 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=61440 expected_recv=48959 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49354 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49380 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49354 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=61440 expected_recv=48959 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=22 calls=1 budget=61440 expected_recv=48959 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=61440 expected_recv=48901 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49354 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49427 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=22 calls=1 budget=61440 expected_recv=48901 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=21 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49427 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49071 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49071 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49327 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=22 calls=1 budget=61440 expected_recv=49327 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=61440 expected_recv=48797 expected_overflow=0 exact_recv_per_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=22 calls=1 budget=61440 expected_recv=48797 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[49460, 49532, 49464, 49195, 48660, 49122, 48705, 49078] avg=49152.0000 max=49532 max_rank=1 min=48660 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[49347, 48863, 49022, 49189, 49417, 49344, 49022, 49012] avg=49152.0000 max=49417 max_rank=4 min=48863 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[49419, 49105, 48925, 49152, 49163, 49249, 49031, 49172] avg=49152.0000 max=49419 max_rank=0 min=48925 min_rank=2 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[49133, 49385, 48704, 49296, 49088, 49216, 49246, 49148] avg=49152.0000 max=49385 max_rank=1 min=48704 min_rank=2 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[48771, 49535, 48586, 49554, 49092, 49190, 49154, 49334] avg=49152.0000 max=49554 max_rank=3 min=48586 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[49152, 49069, 48989, 49463, 48972, 49601, 48987, 48983] avg=49152.0000 max=49601 max_rank=5 min=48972 min_rank=4 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[49092, 49213, 49310, 49264, 49128, 49001, 49335, 48873] avg=49152.0000 max=49335 max_rank=6 min=48873 min_rank=7 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[49652, 48949, 49039, 49106, 49577, 49275, 48873, 48745] avg=49152.0000 max=49652 max_rank=0 min=48745 min_rank=7 imbalance_ratio=1.010173 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[49644, 48949, 49039, 49106, 49577, 49275, 48873, 48753] avg=49152.0000 max=49644 max_rank=0 min=48753 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=8 max_load_reduction_pct=0.000161 moved_to_replicas=8 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[48837, 49366, 49065, 49343, 49170, 49154, 49080, 49201] avg=49152.0000 max=49366 max_rank=1 min=48837 min_rank=0 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=11 tokens_per_ep_rank=[49072, 49055, 49439, 49116, 48811, 49527, 49385, 48811] avg=49152.0000 max=49527 max_rank=5 min=48811 min_rank=4 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=12 tokens_per_ep_rank=[49079, 49265, 49102, 49265, 49236, 49033, 49065, 49171] avg=49152.0000 max=49265 max_rank=1 min=49033 min_rank=5 imbalance_ratio=1.002299 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=13 tokens_per_ep_rank=[49189, 49504, 48983, 49024, 49442, 48941, 49341, 48792] avg=49152.0000 max=49504 max_rank=1 min=48792 min_rank=7 imbalance_ratio=1.007161 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=14 tokens_per_ep_rank=[49185, 49131, 48901, 49401, 49062, 48977, 49139, 49420] avg=49152.0000 max=49420 max_rank=7 min=48901 min_rank=2 imbalance_ratio=1.005452 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=15 tokens_per_ep_rank=[49141, 48797, 49141, 49216, 49282, 49319, 49184, 49136] avg=49152.0000 max=49319 max_rank=5 min=48797 min_rank=1 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=16 tokens_per_ep_rank=[49237, 49136, 49185, 48968, 49120, 48903, 49526, 49141] avg=49152.0000 max=49526 max_rank=6 min=48903 min_rank=5 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=17 tokens_per_ep_rank=[49058, 49416, 49241, 49246, 48987, 49084, 49263, 48921] avg=49152.0000 max=49416 max_rank=1 min=48921 min_rank=7 imbalance_ratio=1.005371 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=18 tokens_per_ep_rank=[49377, 48939, 49070, 49058, 49054, 49600, 49031, 49087] avg=49152.0000 max=49600 max_rank=5 min=48939 min_rank=1 imbalance_ratio=1.009115 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=19 tokens_per_ep_rank=[49024, 48996, 49057, 49243, 49371, 49187, 49218, 49120] avg=49152.0000 max=49371 max_rank=4 min=48996 min_rank=1 imbalance_ratio=1.004456 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=20 tokens_per_ep_rank=[49348, 48801, 49263, 48977, 49091, 49229, 49113, 49394] avg=49152.0000 max=49394 max_rank=7 min=48801 min_rank=1 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=21 tokens_per_ep_rank=[49186, 49431, 48754, 49332, 49027, 49096, 49235, 49155] avg=49152.0000 max=49431 max_rank=1 min=48754 min_rank=2 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=22 tokens_per_ep_rank=[48870, 49088, 48884, 49308, 49144, 49353, 49400, 49169] avg=49152.0000 max=49400 max_rank=6 min=48870 min_rank=0 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=23 total_expected_recv=1128434 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272768 peak_demand=49464 required_capacity=1.006348 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=23 total_expected_recv=1130704 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270592 peak_demand=49526 required_capacity=1.007609 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=23 total_expected_recv=1130448 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49644 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=23 total_expected_recv=1131007 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49535 required_capacity=1.007792 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=23 total_expected_recv=1130378 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271360 peak_demand=49577 required_capacity=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=23 tokens_per_ep_rank=[48926, 49472, 48930, 49226, 49057, 49384, 49017, 49204] avg=49152.0000 max=49472 max_rank=1 min=48926 min_rank=0 imbalance_ratio=1.006510 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=24 tokens_per_ep_rank=[48901, 48959, 49380, 48797, 49427, 49071, 49354, 49327] avg=49152.0000 max=49427 max_rank=4 min=48797 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=23 total_expected_recv=1131856 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268928 peak_demand=49601 required_capacity=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=23 total_expected_recv=1130378 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271360 peak_demand=49577 required_capacity=1.008647 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=23 total_expected_recv=1129402 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272768 peak_demand=49420 required_capacity=1.005452 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=23 total_expected_recv=1128434 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272768 peak_demand=49464 required_capacity=1.006348 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=23 total_expected_recv=1131739 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269440 peak_demand=49554 required_capacity=1.008179 - [2026-08-26 16:49:20] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 625.7 | throughput per GPU (TFLOP/s/GPU): 203.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.944945E+00 | loss scale: 1.0 | grad norm: 41.756 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=61440 expected_recv=48943 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49016 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=23 total_expected_recv=1130704 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270592 peak_demand=49526 required_capacity=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=23 total_expected_recv=1131007 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49535 required_capacity=1.007792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=23 total_expected_recv=1130448 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49644 required_capacity=1.010010 + [2026-08-26 18:01:26] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 627.9 | throughput per GPU (TFLOP/s/GPU): 203.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.944945E+00 | loss scale: 1.0 | grad norm: 41.756 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49273 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49134 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49259 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49304 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=61440 expected_recv=48943 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49016 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49134 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49016 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=61440 expected_recv=48943 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49273 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49134 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49259 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49304 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49134 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49259 expected_overflow=0 exact_recv_per_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49304 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49339 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49144 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49016 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=61440 expected_recv=48943 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=61440 expected_recv=49259 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49056 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49237 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49056 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49339 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49056 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49144 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49339 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49144 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=61440 expected_recv=48834 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49144 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49237 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49056 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=61440 expected_recv=48834 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49050 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49101 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49310 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=61440 expected_recv=48974 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49101 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49101 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49310 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=61440 expected_recv=48974 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49050 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49259 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49050 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=61440 expected_recv=48974 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49310 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49101 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49259 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=61440 expected_recv=48919 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49514 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49215 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=61440 expected_recv=48919 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49117 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49117 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49267 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49514 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49267 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=61440 expected_recv=48919 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49215 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49084 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49514 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=61440 expected_recv=48919 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49215 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49267 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49117 expected_overflow=0 exact_recv_per_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49084 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49084 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=61440 expected_recv=49117 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49552 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49552 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=61440 expected_recv=48910 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=61440 expected_recv=48976 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49151 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49108 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49151 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=61440 expected_recv=48976 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=61440 expected_recv=48910 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=61440 expected_recv=48976 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49552 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49151 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49108 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=61440 expected_recv=48910 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=61440 expected_recv=48976 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49552 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49151 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=61440 expected_recv=48829 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49144 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49147 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=61440 expected_recv=48829 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49144 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=61440 expected_recv=48816 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49147 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49144 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=61440 expected_recv=48829 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49144 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=61440 expected_recv=48829 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49147 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=61440 expected_recv=48816 expected_overflow=0 exact_recv_per_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49166 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=61440 expected_recv=49147 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=61440 expected_recv=48816 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49166 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49126 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49126 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49166 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=61440 expected_recv=48716 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49166 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49139 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=61440 expected_recv=48716 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49379 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=61440 expected_recv=48716 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49139 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49355 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=61440 expected_recv=48716 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49126 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49126 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49379 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49365 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49355 expected_overflow=0 exact_recv_per_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49379 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=61440 expected_recv=48920 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49235 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49365 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=61440 expected_recv=49355 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=61440 expected_recv=48920 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49235 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49365 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49365 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49063 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49230 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49344 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49063 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49235 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49123 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49063 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49230 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=61440 expected_recv=48920 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49344 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49004 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49063 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49235 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49123 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=61440 expected_recv=48920 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49004 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49123 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48994 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=61440 expected_recv=49123 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49139 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48994 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49512 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48677 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49139 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49340 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49090 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49460 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48999 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49512 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48677 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49139 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49340 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48677 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49512 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49340 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49090 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49460 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48999 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49004 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=61440 expected_recv=48677 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49512 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49004 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49542 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49090 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49542 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49090 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49460 expected_overflow=0 exact_recv_per_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49087 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49416 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49453 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=61440 expected_recv=49460 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48712 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49101 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49087 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49416 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48843 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49453 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49416 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49453 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48712 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49101 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48999 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49416 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49453 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48999 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48843 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48918 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48843 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48918 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=61440 expected_recv=48843 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49042 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48978 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49561 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49042 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49282 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48978 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49561 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49101 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48978 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49101 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49561 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49282 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48540 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48540 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=10 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49076 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=10 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49439 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49076 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49494 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48985 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49179 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49076 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49439 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49282 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49076 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49282 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49494 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48985 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49179 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49494 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49179 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48985 expected_overflow=0 exact_recv_per_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=11 calls=1 budget=61440 expected_recv=49494 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49188 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=11 calls=1 budget=61440 expected_recv=48985 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49188 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49188 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49206 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49429 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=61440 expected_recv=48903 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49188 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49363 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49206 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49188 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49429 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=61440 expected_recv=48903 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49363 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49188 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=61440 expected_recv=48903 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49363 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=12 calls=1 budget=61440 expected_recv=48903 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48829 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=12 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48829 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49145 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49313 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49240 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49145 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49429 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49313 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48983 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49145 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49278 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49240 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49429 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48914 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49240 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49174 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48983 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49145 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49278 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48914 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=13 calls=1 budget=61440 expected_recv=49240 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48983 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49174 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49059 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48983 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48914 expected_overflow=0 exact_recv_per_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49059 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49139 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=13 calls=1 budget=61440 expected_recv=48914 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49059 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49225 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49139 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49225 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49278 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=61440 expected_recv=48970 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49337 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=61440 expected_recv=48968 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49139 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49403 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49278 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=14 calls=1 budget=61440 expected_recv=48970 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49337 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=61440 expected_recv=48968 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49139 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=61440 expected_recv=48968 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49403 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=14 calls=1 budget=61440 expected_recv=48968 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49403 expected_overflow=0 exact_recv_per_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49238 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=14 calls=1 budget=61440 expected_recv=49403 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49337 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49238 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49357 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=61440 expected_recv=48976 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49337 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49240 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49357 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49103 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=61440 expected_recv=48976 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49240 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=61440 expected_recv=48976 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49103 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=15 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49238 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=16 calls=1 budget=61440 expected_recv=48976 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49238 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49238 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49261 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49238 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=15 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49261 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49240 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48933 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49240 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=16 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49048 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49141 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48933 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48933 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49048 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49141 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48984 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48933 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49134 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48984 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49241 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48984 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49134 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49134 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=16 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49241 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49134 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49048 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49298 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49048 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49298 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49192 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48934 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49241 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49192 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48934 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49363 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49241 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=17 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49363 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49326 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48934 expected_overflow=0 exact_recv_per_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49363 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49363 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49326 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=17 calls=1 budget=61440 expected_recv=48934 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49161 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49192 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49192 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48763 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=61440 expected_recv=48943 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48920 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49326 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48763 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49101 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48763 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48920 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=61440 expected_recv=48943 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49326 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49259 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=18 calls=1 budget=61440 expected_recv=49101 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48763 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=61440 expected_recv=48943 expected_overflow=0 exact_recv_per_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49259 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49567 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=18 calls=1 budget=61440 expected_recv=48943 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48920 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49567 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49207 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49567 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=61440 expected_recv=48856 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48920 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49207 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49302 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49426 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=61440 expected_recv=48856 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48893 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49302 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=61440 expected_recv=48856 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49259 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49426 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=19 calls=1 budget=61440 expected_recv=48893 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49404 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=20 calls=1 budget=61440 expected_recv=48856 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=61440 expected_recv=48939 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49259 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49426 expected_overflow=0 exact_recv_per_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49404 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=61440 expected_recv=48939 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=19 calls=1 budget=61440 expected_recv=49426 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49302 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=61440 expected_recv=48855 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49299 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=20 calls=1 budget=61440 expected_recv=48939 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49299 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49302 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=61440 expected_recv=48855 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49174 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49174 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=61440 expected_recv=48855 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49299 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49119 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=21 calls=1 budget=61440 expected_recv=48855 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49106 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49119 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=61440 expected_recv=48956 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49119 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49106 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49106 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=61440 expected_recv=48956 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49174 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=20 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49272 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=21 calls=1 budget=61440 expected_recv=48956 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49174 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49272 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49310 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49272 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49315 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49310 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49272 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49315 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=61440 expected_recv=48938 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49038 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49315 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=61440 expected_recv=48938 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49038 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49326 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=61440 expected_recv=48964 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=61440 expected_recv=48964 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49394 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49326 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49310 expected_overflow=0 exact_recv_per_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49038 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=21 calls=1 budget=61440 expected_recv=49310 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49326 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49038 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=61440 expected_recv=48938 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49394 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=22 calls=1 budget=61440 expected_recv=48938 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49394 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=22 calls=1 budget=61440 expected_recv=49326 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=61440 expected_recv=48964 expected_overflow=0 exact_recv_per_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=22 calls=1 budget=61440 expected_recv=48964 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[49259, 49016, 48943, 49116, 49304, 49171, 49273, 49134] avg=49152.0000 max=49304 max_rank=4 min=48943 min_rank=2 imbalance_ratio=1.003092 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[48834, 49144, 49258, 49339, 49078, 49056, 49270, 49237] avg=49152.0000 max=49339 max_rank=3 min=48834 min_rank=0 imbalance_ratio=1.003805 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[49259, 49050, 49132, 49169, 49221, 49101, 49310, 48974] avg=49152.0000 max=49310 max_rank=6 min=48974 min_rank=7 imbalance_ratio=1.003215 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[49084, 49215, 48919, 49514, 49117, 48936, 49164, 49267] avg=49152.0000 max=49514 max_rank=3 min=48919 min_rank=2 imbalance_ratio=1.007365 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[49270, 49171, 49552, 48910, 49151, 49108, 48976, 49078] avg=49152.0000 max=49552 max_rank=2 min=48910 min_rank=3 imbalance_ratio=1.008138 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[48800, 49147, 49660, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49660 max_rank=2 min=48800 min_rank=0 imbalance_ratio=1.010335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[48816, 49147, 49644, 48829, 49057, 49319, 49144, 49260] avg=49152.0000 max=49644 max_rank=2 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[49355, 49379, 49126, 48716, 49263, 49166, 49072, 49139] avg=49152.0000 max=49379 max_rank=1 min=48716 min_rank=3 imbalance_ratio=1.004618 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[48936, 49123, 48920, 49344, 49235, 49365, 49230, 49063] avg=49152.0000 max=49365 max_rank=5 min=48920 min_rank=2 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[49460, 49090, 49004, 49512, 49139, 48994, 48677, 49340] avg=49152.0000 max=49512 max_rank=3 min=48677 min_rank=6 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=11 tokens_per_ep_rank=[49164, 48843, 48999, 49416, 49087, 49542, 49453, 48712] avg=49152.0000 max=49542 max_rank=5 min=48712 min_rank=7 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=12 tokens_per_ep_rank=[48967, 49296, 49101, 48978, 49042, 48918, 49561, 49353] avg=49152.0000 max=49561 max_rank=6 min=48918 min_rank=5 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=13 tokens_per_ep_rank=[48985, 49494, 49282, 49076, 49169, 48540, 49231, 49439] avg=49152.0000 max=49494 max_rank=1 min=48540 min_rank=5 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=14 tokens_per_ep_rank=[49128, 48903, 49164, 49260, 49188, 49179, 49188, 49206] avg=49152.0000 max=49260 max_rank=3 min=48903 min_rank=1 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=15 tokens_per_ep_rank=[48914, 48983, 49429, 49313, 48829, 49363, 49240, 49145] avg=49152.0000 max=49429 max_rank=2 min=48829 min_rank=4 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=16 tokens_per_ep_rank=[49403, 48968, 49278, 49225, 49059, 49174, 48970, 49139] avg=49152.0000 max=49403 max_rank=0 min=48968 min_rank=1 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=17 tokens_per_ep_rank=[48989, 49103, 49337, 49238, 49031, 49091, 49357, 49070] avg=49152.0000 max=49357 max_rank=6 min=48989 min_rank=0 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=18 tokens_per_ep_rank=[49275, 49141, 49240, 49261, 49238, 48976, 48989, 49096] avg=49152.0000 max=49275 max_rank=0 min=48976 min_rank=5 imbalance_ratio=1.002502 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49679, 49048, 49134, 48984, 48898, 49298, 49241] avg=49152.0000 max=49679 max_rank=1 min=48898 min_rank=5 imbalance_ratio=1.010722 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=19 tokens_per_ep_rank=[48934, 49644, 49048, 49134, 48984, 48933, 49298, 49241] avg=49152.0000 max=49644 max_rank=1 min=48933 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=35 max_load_reduction_pct=0.000705 moved_to_replicas=35 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=20 tokens_per_ep_rank=[48943, 49101, 49192, 49161, 49363, 49099, 49031, 49326] avg=49152.0000 max=49363 max_rank=4 min=48943 min_rank=0 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=21 tokens_per_ep_rank=[49426, 48893, 48920, 49567, 49181, 48763, 49207, 49259] avg=49152.0000 max=49567 max_rank=3 min=48763 min_rank=5 imbalance_ratio=1.008443 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=22 tokens_per_ep_rank=[49296, 49119, 49302, 48939, 49404, 48856, 49299, 49001] avg=49152.0000 max=49404 max_rank=4 min=48856 min_rank=5 imbalance_ratio=1.005127 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=23 tokens_per_ep_rank=[49310, 49315, 49174, 48956, 49106, 48855, 49099, 49401] avg=49152.0000 max=49401 max_rank=7 min=48855 min_rank=5 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=23 total_expected_recv=1130977 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269312 peak_demand=49561 required_capacity=1.008321 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=23 total_expected_recv=1131080 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270592 peak_demand=49644 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=23 total_expected_recv=1130414 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271104 peak_demand=49404 required_capacity=1.005127 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=24 tokens_per_ep_rank=[48964, 49394, 49116, 49038, 49168, 49272, 48938, 49326] avg=49152.0000 max=49394 max_rank=1 min=48938 min_rank=6 imbalance_ratio=1.004924 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=23 total_expected_recv=1128777 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273024 peak_demand=49542 required_capacity=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=23 total_expected_recv=1129971 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49460 required_capacity=1.006266 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=23 total_expected_recv=1131011 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270208 peak_demand=49567 required_capacity=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=23 total_expected_recv=1130414 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271104 peak_demand=49404 required_capacity=1.005127 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=23 total_expected_recv=1131206 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270720 peak_demand=49439 required_capacity=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=23 total_expected_recv=1131080 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270592 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=23 total_expected_recv=1131011 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270208 peak_demand=49567 required_capacity=1.008443 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=23 total_expected_recv=1130977 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269312 peak_demand=49561 required_capacity=1.008321 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=23 total_expected_recv=1130532 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270592 peak_demand=49644 required_capacity=1.010010 - [2026-08-26 16:49:21] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 624.3 | throughput per GPU (TFLOP/s/GPU): 204.4 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.474912E+00 | loss scale: 1.0 | grad norm: 7.074 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=23 total_expected_recv=1129971 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49460 required_capacity=1.006266 + [2026-08-26 18:01:27] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 626.3 | throughput per GPU (TFLOP/s/GPU): 203.7 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.474912E+00 | loss scale: 1.0 | grad norm: 7.074 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48920 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48998 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49234 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48983 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49442 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48887 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48998 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48920 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49234 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49108 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49234 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48998 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48920 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48983 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49442 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48887 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48887 expected_overflow=0 exact_recv_per_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49108 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49234 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48998 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=61440 expected_recv=48887 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49247 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49143 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=61440 expected_recv=48879 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49247 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=61440 expected_recv=48885 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49143 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=61440 expected_recv=48879 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49247 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=61440 expected_recv=48885 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=61440 expected_recv=48879 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49466 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49209 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49143 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=61440 expected_recv=48879 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49123 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=61440 expected_recv=48885 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49466 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49209 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49143 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49123 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49466 expected_overflow=0 exact_recv_per_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49507 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49073 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=61440 expected_recv=49466 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=61440 expected_recv=48886 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49507 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49073 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=61440 expected_recv=48886 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=61440 expected_recv=48943 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49129 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49054 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=61440 expected_recv=48886 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=61440 expected_recv=48943 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49507 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=61440 expected_recv=48886 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49419 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49054 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49129 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49507 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49419 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49054 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49129 expected_overflow=0 exact_recv_per_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49419 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49311 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48933 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=61440 expected_recv=49129 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49311 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48933 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49004 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48985 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48933 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49004 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49253 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48985 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=61440 expected_recv=48933 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49253 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49218 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49147 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49218 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49147 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49191 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49073 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49191 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49147 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49244 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49073 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49147 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49244 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49501 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=61440 expected_recv=48835 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49501 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49244 expected_overflow=0 exact_recv_per_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=61440 expected_recv=48835 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49501 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=61440 expected_recv=48835 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=61440 expected_recv=49244 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49516 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49516 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49197 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49197 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=61440 expected_recv=48976 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49194 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=61440 expected_recv=48976 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49197 expected_overflow=0 exact_recv_per_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49437 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=61440 expected_recv=48976 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49088 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49194 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=61440 expected_recv=48976 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49437 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=61440 expected_recv=49197 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49088 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49088 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49088 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49437 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=61440 expected_recv=48986 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49437 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=61440 expected_recv=48985 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=61440 expected_recv=48986 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=61440 expected_recv=48986 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=61440 expected_recv=48896 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=61440 expected_recv=48985 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49015 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=61440 expected_recv=48986 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=61440 expected_recv=48896 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49015 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49015 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49471 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49015 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49085 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49471 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=61440 expected_recv=48912 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49085 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=61440 expected_recv=48912 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49161 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49253 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=61440 expected_recv=48912 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49252 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49161 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49085 expected_overflow=0 exact_recv_per_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49253 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=61440 expected_recv=48912 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49252 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=61440 expected_recv=49085 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49281 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49252 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=61440 expected_recv=48977 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49281 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49252 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49056 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=61440 expected_recv=48977 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49056 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=61440 expected_recv=48879 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49056 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49405 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=61440 expected_recv=48879 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=61440 expected_recv=48977 expected_overflow=0 exact_recv_per_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=61440 expected_recv=49056 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49229 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49405 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49219 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49405 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=61440 expected_recv=48977 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49229 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49212 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49219 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49405 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49332 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49212 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49332 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=61440 expected_recv=48896 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49147 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49332 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=61440 expected_recv=48896 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48820 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=61440 expected_recv=48896 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49147 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49212 expected_overflow=0 exact_recv_per_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=61440 expected_recv=48896 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49332 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49006 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48820 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49006 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=61440 expected_recv=49212 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48808 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49601 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49006 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48808 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49601 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49379 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49423 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49379 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49379 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49601 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49423 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49379 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48808 expected_overflow=0 exact_recv_per_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=10 calls=1 budget=61440 expected_recv=49601 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=10 calls=1 budget=61440 expected_recv=48808 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=61440 expected_recv=48953 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=61440 expected_recv=48962 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49497 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=11 calls=1 budget=61440 expected_recv=48953 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=61440 expected_recv=48831 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=11 calls=1 budget=61440 expected_recv=48962 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49497 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=61440 expected_recv=48831 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49203 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49055 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49203 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49203 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=61440 expected_recv=48831 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49055 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=61440 expected_recv=48921 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49532 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49203 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=11 calls=1 budget=61440 expected_recv=48831 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49497 expected_overflow=0 exact_recv_per_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49055 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=12 calls=1 budget=61440 expected_recv=48921 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=11 calls=1 budget=61440 expected_recv=49497 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49055 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49532 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49300 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49345 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49429 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49300 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49300 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49532 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49345 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49429 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49300 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=12 calls=1 budget=61440 expected_recv=49532 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49052 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49429 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=12 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49135 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49429 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=61440 expected_recv=48965 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49135 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=13 calls=1 budget=61440 expected_recv=48965 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49135 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=61440 expected_recv=48871 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49052 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49291 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49135 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=61440 expected_recv=48871 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49291 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=61440 expected_recv=48851 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=61440 expected_recv=48851 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49566 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49291 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=61440 expected_recv=48871 expected_overflow=0 exact_recv_per_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=13 calls=1 budget=61440 expected_recv=49291 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=61440 expected_recv=48851 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=13 calls=1 budget=61440 expected_recv=48871 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=14 calls=1 budget=61440 expected_recv=48851 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49566 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=61440 expected_recv=48592 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49566 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=61440 expected_recv=48592 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49320 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=61440 expected_recv=48986 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49320 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49109 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=61440 expected_recv=48986 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49320 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=61440 expected_recv=48592 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49109 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49368 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=61440 expected_recv=48986 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=15 calls=1 budget=61440 expected_recv=48592 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49368 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=14 calls=1 budget=61440 expected_recv=48986 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49321 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=14 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49109 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49321 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49109 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49360 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49321 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49360 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49040 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49239 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=61440 expected_recv=48778 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49040 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49239 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49239 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49253 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=61440 expected_recv=48778 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49360 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49239 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49253 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49360 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49040 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49191 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49345 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49253 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49040 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=15 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=61440 expected_recv=48778 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49047 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49191 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49345 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49047 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49280 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=16 calls=1 budget=61440 expected_recv=48778 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49047 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49359 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49470 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49280 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49359 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49359 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48957 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49470 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48849 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49347 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49345 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49359 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48957 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48849 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49347 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49345 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49280 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49347 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49280 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=16 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49347 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49470 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49480 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49012 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49470 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49480 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=61440 expected_recv=48929 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49012 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49277 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=61440 expected_recv=48929 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49119 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49277 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49325 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49306 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49277 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49119 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=61440 expected_recv=48916 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49325 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=17 calls=1 budget=61440 expected_recv=49277 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49012 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49306 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=61440 expected_recv=48916 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49306 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48849 expected_overflow=0 exact_recv_per_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49012 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49157 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=61440 expected_recv=48929 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49306 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=17 calls=1 budget=61440 expected_recv=48849 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49157 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=18 calls=1 budget=61440 expected_recv=48929 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49247 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49129 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49247 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49055 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49129 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=61440 expected_recv=48944 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49129 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=61440 expected_recv=48916 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49055 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49300 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49239 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=61440 expected_recv=48944 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49129 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=18 calls=1 budget=61440 expected_recv=48916 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49300 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49239 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49300 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49325 expected_overflow=0 exact_recv_per_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49145 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49247 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49300 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=18 calls=1 budget=61440 expected_recv=49325 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49145 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49247 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=61440 expected_recv=48907 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49063 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=61440 expected_recv=48907 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49124 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49063 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49106 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49239 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49063 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49124 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49239 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=19 calls=1 budget=61440 expected_recv=49063 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49369 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49175 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49106 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49369 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49175 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=61440 expected_recv=48944 expected_overflow=0 exact_recv_per_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=61440 expected_recv=48863 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49028 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49369 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=61440 expected_recv=48907 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=19 calls=1 budget=61440 expected_recv=48944 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49343 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=21 calls=1 budget=61440 expected_recv=48863 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49028 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49133 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=20 calls=1 budget=61440 expected_recv=48907 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49343 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49411 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49133 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49175 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49133 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49411 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49175 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49028 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49133 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49028 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49015 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49133 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49255 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49106 expected_overflow=0 exact_recv_per_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49015 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49133 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49255 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49343 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=20 calls=1 budget=61440 expected_recv=49106 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49153 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49466 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49153 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49133 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49343 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49466 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49133 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49153 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49153 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=61440 expected_recv=48851 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=22 calls=1 budget=61440 expected_recv=48851 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=21 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=61440 expected_recv=48929 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=22 calls=1 budget=61440 expected_recv=48929 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=22 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[48885, 49646, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49646 max_rank=1 min=48885 min_rank=0 imbalance_ratio=1.010050 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[48887, 49644, 49234, 48998, 49442, 48920, 48983, 49108] avg=49152.0000 max=49644 max_rank=1 min=48887 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[49123, 49466, 48879, 49143, 49264, 49247, 48885, 49209] avg=49152.0000 max=49466 max_rank=1 min=48879 min_rank=2 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[49419, 49129, 48943, 49205, 49507, 49073, 48886, 49054] avg=49152.0000 max=49507 max_rank=4 min=48886 min_rank=6 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[49253, 49323, 49004, 48985, 49111, 49311, 48933, 49296] avg=49152.0000 max=49323 max_rank=1 min=48933 min_rank=6 imbalance_ratio=1.003479 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[49244, 48835, 49191, 49073, 49007, 49218, 49147, 49501] avg=49152.0000 max=49501 max_rank=7 min=48835 min_rank=1 imbalance_ratio=1.007100 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[49197, 49194, 49099, 49516, 49078, 49057, 49099, 48976] avg=49152.0000 max=49516 max_rank=3 min=48976 min_rank=7 imbalance_ratio=1.007406 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[49181, 48985, 49088, 49377, 49437, 49111, 49051, 48986] avg=49152.0000 max=49437 max_rank=4 min=48985 min_rank=1 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[49085, 49161, 49015, 49471, 49353, 48896, 49323, 48912] avg=49152.0000 max=49471 max_rank=3 min=48896 min_rank=5 imbalance_ratio=1.006490 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[48977, 49195, 49066, 49136, 49252, 49253, 49281, 49056] avg=49152.0000 max=49281 max_rank=6 min=48977 min_rank=0 imbalance_ratio=1.002625 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=11 tokens_per_ep_rank=[49212, 48896, 49044, 49219, 49405, 48879, 49229, 49332] avg=49152.0000 max=49405 max_rank=4 min=48879 min_rank=5 imbalance_ratio=1.005147 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=12 tokens_per_ep_rank=[48808, 49379, 48820, 49006, 49136, 49147, 49319, 49601] avg=49152.0000 max=49601 max_rank=7 min=48808 min_rank=0 imbalance_ratio=1.009135 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=13 tokens_per_ep_rank=[49497, 49203, 49296, 48962, 49051, 49423, 48953, 48831] avg=49152.0000 max=49497 max_rank=0 min=48831 min_rank=7 imbalance_ratio=1.007019 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=14 tokens_per_ep_rank=[48955, 49300, 49055, 49384, 48921, 49003, 49066, 49532] avg=49152.0000 max=49532 max_rank=7 min=48921 min_rank=4 imbalance_ratio=1.007731 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=15 tokens_per_ep_rank=[48871, 49128, 49429, 49135, 49052, 49345, 48965, 49291] avg=49152.0000 max=49429 max_rank=2 min=48871 min_rank=0 imbalance_ratio=1.005636 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=16 tokens_per_ep_rank=[49275, 48986, 48851, 49024, 49116, 49078, 49566, 49320] avg=49152.0000 max=49566 max_rank=6 min=48851 min_rank=2 imbalance_ratio=1.008423 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=17 tokens_per_ep_rank=[49226, 49239, 49368, 49321, 49109, 48592, 49001, 49360] avg=49152.0000 max=49368 max_rank=2 min=48592 min_rank=5 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=18 tokens_per_ep_rank=[49079, 49359, 49040, 49191, 49253, 49171, 48778, 49345] avg=49152.0000 max=49359 max_rank=1 min=48778 min_rank=6 imbalance_ratio=1.004211 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=19 tokens_per_ep_rank=[48849, 49277, 49280, 49347, 48957, 49047, 49470, 48989] avg=49152.0000 max=49470 max_rank=6 min=48849 min_rank=0 imbalance_ratio=1.006470 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=20 tokens_per_ep_rank=[49325, 49129, 49012, 49306, 49119, 49480, 48929, 48916] avg=49152.0000 max=49480 max_rank=5 min=48916 min_rank=7 imbalance_ratio=1.006673 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=21 tokens_per_ep_rank=[48944, 49063, 49211, 49300, 49055, 49157, 49247, 49239] avg=49152.0000 max=49300 max_rank=3 min=48944 min_rank=0 imbalance_ratio=1.003011 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=22 tokens_per_ep_rank=[49106, 49133, 49257, 49369, 49124, 49145, 48907, 49175] avg=49152.0000 max=49369 max_rank=3 min=48907 min_rank=6 imbalance_ratio=1.004415 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=23 tokens_per_ep_rank=[49270, 49153, 49028, 49015, 49411, 48863, 49343, 49133] avg=49152.0000 max=49411 max_rank=4 min=48863 min_rank=5 imbalance_ratio=1.005269 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=24 tokens_per_ep_rank=[49169, 48929, 49173, 48851, 49466, 49255, 49077, 49296] avg=49152.0000 max=49466 max_rank=4 min=48851 min_rank=3 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=23 total_expected_recv=1129671 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271744 peak_demand=49480 required_capacity=1.006673 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=23 total_expected_recv=1129438 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49566 required_capacity=1.008423 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=23 total_expected_recv=1129383 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271616 peak_demand=49429 required_capacity=1.005636 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=23 total_expected_recv=1131626 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269952 peak_demand=49507 required_capacity=1.007222 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=23 total_expected_recv=1129952 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270720 peak_demand=49497 required_capacity=1.007019 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=23 total_expected_recv=1129383 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271616 peak_demand=49429 required_capacity=1.005636 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=23 total_expected_recv=1131458 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49601 required_capacity=1.009135 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=23 total_expected_recv=1131334 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269952 peak_demand=49516 required_capacity=1.007406 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=23 total_expected_recv=1129438 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49566 required_capacity=1.008423 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=23 total_expected_recv=1131106 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49644 required_capacity=1.010010 - [2026-08-26 16:49:21] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 625.4 | throughput per GPU (TFLOP/s/GPU): 204.0 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.119712E+01 | loss scale: 1.0 | grad norm: 398.552 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=23 total_expected_recv=1129952 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270720 peak_demand=49497 required_capacity=1.007019 + [2026-08-26 18:01:27] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 629.6 | throughput per GPU (TFLOP/s/GPU): 202.7 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.119712E+01 | loss scale: 1.0 | grad norm: 398.552 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49276 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=61440 expected_recv=48938 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49009 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49438 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=61440 expected_recv=48880 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=61440 expected_recv=48938 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49276 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49438 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49009 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49276 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=61440 expected_recv=48938 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49009 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49438 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=61440 expected_recv=48999 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=61440 expected_recv=48880 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=61440 expected_recv=48938 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=61440 expected_recv=48999 expected_overflow=0 exact_recv_per_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49276 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49438 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=61440 expected_recv=49009 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=61440 expected_recv=48999 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49244 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49505 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=61440 expected_recv=48909 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49008 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49244 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49505 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49332 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=61440 expected_recv=48909 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49008 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49332 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49008 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49332 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49008 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49244 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49332 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49505 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49244 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49505 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49223 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49223 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=61440 expected_recv=48788 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=61440 expected_recv=48788 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49274 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=61440 expected_recv=48810 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49274 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49223 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=61440 expected_recv=48810 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49399 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=61440 expected_recv=48788 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49223 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50304 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49244 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49399 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49244 expected_overflow=0 exact_recv_per_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=61440 expected_recv=48788 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=61440 expected_recv=49244 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49337 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=61440 expected_recv=48773 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49337 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=61440 expected_recv=48773 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49068 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49071 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49060 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49068 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49399 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49071 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49060 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49340 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=61440 expected_recv=48773 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49337 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49068 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49060 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49340 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49399 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49071 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49060 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=61440 expected_recv=48748 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49071 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49340 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=61440 expected_recv=48773 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49337 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=61440 expected_recv=48748 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49182 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49182 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49196 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49358 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49100 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49196 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49358 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=61440 expected_recv=48748 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49439 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49100 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49100 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49182 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=61440 expected_recv=48748 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49196 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49439 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49100 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49115 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49266 expected_overflow=0 exact_recv_per_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49182 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49196 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49201 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49115 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=61440 expected_recv=49266 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49201 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49626 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49626 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49115 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=61440 expected_recv=48988 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49201 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49115 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49626 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49387 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48924 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=61440 expected_recv=48855 expected_overflow=0 exact_recv_per_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49201 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49626 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49236 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49387 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48924 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=61440 expected_recv=48855 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49236 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48929 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49387 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48929 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49236 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49387 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48924 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48765 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49236 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48929 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48924 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48765 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48765 expected_overflow=0 exact_recv_per_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=61440 expected_recv=48765 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49321 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49053 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49015 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49456 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49321 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49053 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49456 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49015 expected_overflow=0 exact_recv_per_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49015 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49232 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48893 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49232 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48893 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49456 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49565 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49456 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49296 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49565 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49232 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=61440 expected_recv=49015 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49232 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48893 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49565 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48893 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48871 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49198 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49296 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49185 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49131 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=61440 expected_recv=49565 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48871 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49185 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49198 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49185 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49131 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49185 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49131 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48945 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49131 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49328 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49198 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49198 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49151 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48945 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=61440 expected_recv=48990 expected_overflow=0 exact_recv_per_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49176 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48952 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49328 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=61440 expected_recv=49151 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48952 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=61440 expected_recv=48990 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49176 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48952 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49328 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49328 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49176 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49176 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49018 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=61440 expected_recv=48843 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48895 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48895 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48945 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49436 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48945 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49311 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49018 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=61440 expected_recv=48843 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49418 expected_overflow=0 exact_recv_per_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=61440 expected_recv=48985 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=10 calls=1 budget=61440 expected_recv=48895 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49436 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49311 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=10 calls=1 budget=61440 expected_recv=49418 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=11 calls=1 budget=61440 expected_recv=48985 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=61440 expected_recv=48843 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49311 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49311 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49343 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49067 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49430 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=11 calls=1 budget=61440 expected_recv=48843 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49430 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49018 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49018 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49067 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49343 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49150 expected_overflow=0 exact_recv_per_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=61440 expected_recv=48960 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49343 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49430 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=11 calls=1 budget=61440 expected_recv=49150 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=12 calls=1 budget=61440 expected_recv=48960 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49343 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49067 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49067 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49290 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49561 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49129 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49290 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49561 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49129 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=61440 expected_recv=48938 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49286 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49427 expected_overflow=0 exact_recv_per_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49129 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49286 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=12 calls=1 budget=61440 expected_recv=48938 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49286 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=12 calls=1 budget=61440 expected_recv=49427 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49129 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49286 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49561 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49290 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49561 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49290 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49474 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49474 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=61440 expected_recv=48719 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=61440 expected_recv=48719 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=61440 expected_recv=48882 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49015 expected_overflow=0 exact_recv_per_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=13 calls=1 budget=61440 expected_recv=48882 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49375 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=13 calls=1 budget=61440 expected_recv=49015 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49474 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=61440 expected_recv=48719 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49375 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=14 calls=1 budget=61440 expected_recv=48719 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49100 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49100 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=61440 expected_recv=48632 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49474 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49503 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=61440 expected_recv=48632 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49301 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49503 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49301 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49375 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49113 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49259 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49113 expected_overflow=0 exact_recv_per_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49301 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49386 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49375 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49113 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49259 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=14 calls=1 budget=61440 expected_recv=49113 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49301 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49503 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49100 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=61440 expected_recv=48632 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49386 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49503 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=15 calls=1 budget=61440 expected_recv=48632 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=61440 expected_recv=48800 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=61440 expected_recv=48800 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49045 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49113 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49100 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49113 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49399 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49577 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49399 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49577 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49386 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49045 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49386 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49161 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49045 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49577 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=15 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=61440 expected_recv=48800 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49399 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49113 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49399 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49113 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49090 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49090 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49249 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=16 calls=1 budget=61440 expected_recv=48800 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=61440 expected_recv=48788 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49249 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=61440 expected_recv=48984 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=61440 expected_recv=48788 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=61440 expected_recv=48984 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=61440 expected_recv=48948 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49460 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=61440 expected_recv=48984 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=16 calls=1 budget=61440 expected_recv=48948 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49460 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=17 calls=1 budget=61440 expected_recv=48984 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49063 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49090 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=61440 expected_recv=48956 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=16 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=61440 expected_recv=48788 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49063 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49249 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=17 calls=1 budget=61440 expected_recv=48788 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49249 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49062 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49090 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=61440 expected_recv=48956 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49010 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49062 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49010 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49517 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49063 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49065 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49517 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49358 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49010 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49063 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49065 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49358 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49517 expected_overflow=0 exact_recv_per_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49010 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49365 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49121 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49062 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=17 calls=1 budget=61440 expected_recv=49517 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=61440 expected_recv=48956 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49365 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49062 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=18 calls=1 budget=61440 expected_recv=48956 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=61440 expected_recv=48986 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49266 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49254 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=61440 expected_recv=48986 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49266 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49121 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49254 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49121 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49056 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49365 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49121 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49056 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=61440 expected_recv=48986 expected_overflow=0 exact_recv_per_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49266 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49153 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48993 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=18 calls=1 budget=61440 expected_recv=49365 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48972 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=18 calls=1 budget=61440 expected_recv=48986 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49266 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48993 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49232 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49153 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48993 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48972 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48993 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49232 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49193 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48993 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49265 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48972 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48993 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49193 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49265 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48972 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49153 expected_overflow=0 exact_recv_per_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49056 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48993 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=19 calls=1 budget=61440 expected_recv=49153 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49056 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48902 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48905 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49546 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49056 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=20 calls=1 budget=61440 expected_recv=48993 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49056 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48902 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48905 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49546 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48902 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48902 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48857 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49056 expected_overflow=0 exact_recv_per_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=20 calls=1 budget=61440 expected_recv=49056 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49265 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49546 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=61440 expected_recv=48734 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49492 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48857 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49265 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49546 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=61440 expected_recv=48734 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49492 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49199 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49434 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=61440 expected_recv=48734 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49199 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49434 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=22 calls=1 budget=61440 expected_recv=48734 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49375 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49375 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48857 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=21 calls=1 budget=61440 expected_recv=48857 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=21 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49375 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49199 expected_overflow=0 exact_recv_per_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49375 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=22 calls=1 budget=61440 expected_recv=49199 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[48999, 49384, 49292, 48880, 49009, 49276, 49438, 48938] avg=49152.0000 max=49438 max_rank=6 min=48880 min_rank=3 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[49051, 49231, 49505, 49244, 48936, 48909, 49332, 49008] avg=49152.0000 max=49505 max_rank=2 min=48909 min_rank=5 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49662, 48810, 48770, 49223, 49154, 49079] avg=49152.0000 max=49662 max_rank=2 min=48770 min_rank=4 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[49244, 49274, 49644, 48810, 48788, 49223, 49154, 49079] avg=49152.0000 max=49644 max_rank=2 min=48788 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=18 max_load_reduction_pct=0.000362 moved_to_replicas=18 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[49168, 49068, 49071, 49060, 49337, 49399, 49340, 48773] avg=49152.0000 max=49399 max_rank=5 min=48773 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[49266, 49358, 49196, 49100, 49182, 48748, 49439, 48927] avg=49152.0000 max=49439 max_rank=6 min=48748 min_rank=5 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[48855, 49164, 49626, 49264, 49201, 49115, 48988, 49003] avg=49152.0000 max=49626 max_rank=2 min=48855 min_rank=0 imbalance_ratio=1.009644 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[48765, 49401, 49384, 49190, 49236, 49387, 48929, 48924] avg=49152.0000 max=49401 max_rank=1 min=48765 min_rank=0 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49756, 48877, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49756 max_rank=1 min=48877 min_rank=2 imbalance_ratio=1.012288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[49015, 49644, 48989, 49053, 49171, 49001, 49321, 49022] avg=49152.0000 max=49644 max_rank=1 min=48989 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=112 max_load_reduction_pct=0.002251 moved_to_replicas=112 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[48967, 49232, 48893, 48871, 48936, 49456, 49565, 49296] avg=49152.0000 max=49565 max_rank=6 min=48871 min_rank=3 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=11 tokens_per_ep_rank=[48990, 49185, 49131, 49151, 49198, 49264, 49070, 49227] avg=49152.0000 max=49264 max_rank=5 min=48990 min_rank=0 imbalance_ratio=1.002279 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=12 tokens_per_ep_rank=[49418, 48952, 49176, 49436, 49066, 49328, 48895, 48945] avg=49152.0000 max=49436 max_rank=3 min=48895 min_rank=6 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=13 tokens_per_ep_rank=[49150, 48985, 49311, 49292, 49187, 48843, 49430, 49018] avg=49152.0000 max=49430 max_rank=6 min=48843 min_rank=5 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=14 tokens_per_ep_rank=[49427, 48960, 49257, 48938, 49067, 49343, 49221, 49003] avg=49152.0000 max=49427 max_rank=0 min=48938 min_rank=3 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=15 tokens_per_ep_rank=[49015, 49286, 49031, 48882, 49561, 49129, 49022, 49290] avg=49152.0000 max=49561 max_rank=4 min=48882 min_rank=3 imbalance_ratio=1.008321 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=16 tokens_per_ep_rank=[49113, 49079, 49474, 49259, 48719, 49007, 49375, 49190] avg=49152.0000 max=49474 max_rank=2 min=48719 min_rank=4 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=17 tokens_per_ep_rank=[49022, 49111, 49100, 49161, 49503, 49301, 49386, 48632] avg=49152.0000 max=49503 max_rank=4 min=48632 min_rank=7 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=18 tokens_per_ep_rank=[49070, 49045, 48800, 48948, 49399, 49577, 49264, 49113] avg=49152.0000 max=49577 max_rank=5 min=48800 min_rank=2 imbalance_ratio=1.008647 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=19 tokens_per_ep_rank=[49517, 49460, 49090, 49065, 48788, 48984, 49063, 49249] avg=49152.0000 max=49517 max_rank=0 min=48788 min_rank=4 imbalance_ratio=1.007426 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=20 tokens_per_ep_rank=[49258, 49358, 49221, 48986, 49062, 49010, 49365, 48956] avg=49152.0000 max=49365 max_rank=6 min=48956 min_rank=7 imbalance_ratio=1.004333 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=21 tokens_per_ep_rank=[49193, 49266, 49019, 49153, 49121, 49254, 49177, 49033] avg=49152.0000 max=49266 max_rank=1 min=49019 min_rank=2 imbalance_ratio=1.002319 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[49688, 48993, 49056, 49056, 48949, 49232, 49270, 48972] avg=49152.0000 max=49688 max_rank=0 min=48949 min_rank=4 imbalance_ratio=1.010905 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=22 tokens_per_ep_rank=[49644, 48993, 49056, 49056, 48993, 49232, 49270, 48972] avg=49152.0000 max=49644 max_rank=0 min=48972 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=44 max_load_reduction_pct=0.000886 moved_to_replicas=44 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=23 tokens_per_ep_rank=[49434, 49546, 49265, 48857, 49171, 48905, 49136, 48902] avg=49152.0000 max=49546 max_rank=1 min=48857 min_rank=3 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=23 total_expected_recv=1131636 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269696 peak_demand=49644 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=23 total_expected_recv=1129768 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49561 required_capacity=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=24 tokens_per_ep_rank=[49069, 49105, 49105, 49375, 49137, 49492, 49199, 48734] avg=49152.0000 max=49492 max_rank=5 min=48734 min_rank=7 imbalance_ratio=1.006917 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=23 total_expected_recv=1127234 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=274688 peak_demand=49296 required_capacity=1.002930 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=23 total_expected_recv=1131183 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269696 peak_demand=49577 required_capacity=1.008647 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=23 total_expected_recv=1132379 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268544 peak_demand=49565 required_capacity=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=23 total_expected_recv=1129768 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49561 required_capacity=1.008321 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=23 total_expected_recv=1131636 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269696 peak_demand=49644 required_capacity=1.010010 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=23 total_expected_recv=1132087 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269312 peak_demand=49644 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=23 total_expected_recv=1127234 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=274688 peak_demand=49296 required_capacity=1.002930 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=23 total_expected_recv=1130650 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49644 required_capacity=1.010010 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=23 total_expected_recv=1129031 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272512 peak_demand=49436 required_capacity=1.005778 - [2026-08-26 16:49:22] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 624.2 | throughput per GPU (TFLOP/s/GPU): 204.4 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.384323E+00 | loss scale: 1.0 | grad norm: 5.040 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49008 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=23 total_expected_recv=1132379 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268544 peak_demand=49565 required_capacity=1.008403 + [2026-08-26 18:01:28] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 626.3 | throughput per GPU (TFLOP/s/GPU): 203.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.384323E+00 | loss scale: 1.0 | grad norm: 5.040 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49408 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49008 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49032 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49216 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49408 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49573 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49008 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48827 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49408 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48827 expected_overflow=0 exact_recv_per_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49008 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49032 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49216 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49408 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=61440 expected_recv=49573 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48827 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=61440 expected_recv=48827 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49269 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=61440 expected_recv=48925 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49269 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49052 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49269 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49307 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49141 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=61440 expected_recv=48925 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49340 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49110 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49269 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49340 expected_overflow=0 exact_recv_per_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49141 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49307 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49052 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49141 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=61440 expected_recv=49340 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49148 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49038 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49038 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49393 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49148 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=61440 expected_recv=48962 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49038 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49148 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49393 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=61440 expected_recv=48962 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49038 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49269 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49148 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49108 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49269 expected_overflow=0 exact_recv_per_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49082 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=61440 expected_recv=49269 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49082 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49082 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48858 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48906 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48858 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49082 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48858 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48906 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49550 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=61440 expected_recv=48906 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49550 expected_overflow=0 exact_recv_per_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=61440 expected_recv=49550 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49274 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49274 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49085 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49026 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49274 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49192 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49085 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49026 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49110 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49209 expected_overflow=0 exact_recv_per_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49167 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49192 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49110 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=61440 expected_recv=49209 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49167 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49355 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49355 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49167 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=61440 expected_recv=48797 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49008 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49355 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49379 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49167 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49220 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=61440 expected_recv=48797 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49008 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49379 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=61440 expected_recv=48646 expected_overflow=0 exact_recv_per_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49379 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49220 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=61440 expected_recv=49379 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=61440 expected_recv=48646 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49299 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49299 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48925 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49354 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49299 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49071 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=61440 expected_recv=48925 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49354 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49030 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49352 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49354 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49071 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49071 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49290 expected_overflow=0 exact_recv_per_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49030 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49352 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49071 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=61440 expected_recv=49290 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=61440 expected_recv=48857 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49458 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=61440 expected_recv=48857 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49458 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49352 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49118 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49458 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49352 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49458 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49248 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=61440 expected_recv=48993 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=61440 expected_recv=48717 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49118 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49248 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49291 expected_overflow=0 exact_recv_per_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49118 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=61440 expected_recv=48993 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=61440 expected_recv=48717 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=61440 expected_recv=48993 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49118 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=61440 expected_recv=49291 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49453 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49161 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49453 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=61440 expected_recv=48717 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=61440 expected_recv=48717 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49453 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49040 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49028 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49248 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49537 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49398 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49453 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49040 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49028 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49248 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49537 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49032 expected_overflow=0 exact_recv_per_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49537 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49028 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49398 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=61440 expected_recv=48937 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49537 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49032 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=61440 expected_recv=49032 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=61440 expected_recv=48937 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49207 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=61440 expected_recv=48937 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49207 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49398 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49398 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49297 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48885 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49207 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49219 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49071 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49053 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49297 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48885 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49207 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49071 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49219 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49071 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49053 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49249 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49071 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49249 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49053 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49249 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49366 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49249 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49366 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49053 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48885 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49042 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49366 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49467 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49172 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48827 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48885 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49042 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49366 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49467 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48827 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49172 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48827 expected_overflow=0 exact_recv_per_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49509 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=10 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49509 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=10 calls=1 budget=61440 expected_recv=48827 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49172 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49509 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49172 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49256 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49042 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48900 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49256 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49042 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=61440 expected_recv=48801 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48900 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=61440 expected_recv=48801 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49224 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48970 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=11 calls=1 budget=61440 expected_recv=48801 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49224 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48970 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=11 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49439 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49127 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48900 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49127 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49220 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49439 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49127 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48970 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49220 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48900 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49239 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49220 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48892 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49414 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49239 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48892 expected_overflow=0 exact_recv_per_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48892 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48970 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=12 calls=1 budget=61440 expected_recv=49239 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49414 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=12 calls=1 budget=61440 expected_recv=48892 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49241 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49036 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49439 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49036 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=61440 expected_recv=48810 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49439 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49241 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49036 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=61440 expected_recv=48810 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49029 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=61440 expected_recv=48858 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49318 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=61440 expected_recv=48858 expected_overflow=0 exact_recv_per_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=13 calls=1 budget=61440 expected_recv=49029 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=61440 expected_recv=48858 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49318 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=13 calls=1 budget=61440 expected_recv=48858 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=61440 expected_recv=48810 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49278 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=14 calls=1 budget=61440 expected_recv=48810 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49238 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49241 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49238 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49278 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=61440 expected_recv=48916 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=61440 expected_recv=48916 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49241 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49348 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49238 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=14 calls=1 budget=61440 expected_recv=48916 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49298 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=61440 expected_recv=48916 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=61440 expected_recv=48901 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49118 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49348 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49298 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49118 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=61440 expected_recv=48901 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49348 expected_overflow=0 exact_recv_per_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49118 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=15 calls=1 budget=61440 expected_recv=48901 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=14 calls=1 budget=61440 expected_recv=49348 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=61440 expected_recv=48916 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49108 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49278 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48966 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=15 calls=1 budget=61440 expected_recv=48916 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49108 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49350 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49278 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49254 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49432 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48966 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49108 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49350 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49322 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49254 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48910 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49432 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49084 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49298 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48910 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49254 expected_overflow=0 exact_recv_per_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49298 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48910 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49084 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=15 calls=1 budget=61440 expected_recv=49254 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48966 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49358 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49032 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=16 calls=1 budget=61440 expected_recv=48966 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49338 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49084 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49358 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49322 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49032 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49393 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49338 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49084 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49432 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49393 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=61440 expected_recv=48854 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49393 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49432 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=61440 expected_recv=48854 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49393 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49358 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=61440 expected_recv=48829 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=16 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49064 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49231 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49309 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49358 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49438 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=61440 expected_recv=48829 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=61440 expected_recv=48804 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=61440 expected_recv=48854 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49064 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49231 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49309 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49438 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=17 calls=1 budget=61440 expected_recv=48854 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=61440 expected_recv=48804 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49217 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49338 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49368 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49217 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49338 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49217 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49368 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=61440 expected_recv=48804 expected_overflow=0 exact_recv_per_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49430 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49064 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49419 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49013 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49064 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49288 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49419 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49083 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=17 calls=1 budget=61440 expected_recv=49217 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=61440 expected_recv=48829 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=17 calls=1 budget=61440 expected_recv=48804 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49430 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49013 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49288 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49402 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=18 calls=1 budget=61440 expected_recv=48829 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49083 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49438 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=61440 expected_recv=48605 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49402 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49438 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=19 calls=1 budget=61440 expected_recv=48605 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49430 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49402 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49528 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49083 expected_overflow=0 exact_recv_per_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49229 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49430 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49458 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=61440 expected_recv=48960 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49402 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49419 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49528 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=18 calls=1 budget=61440 expected_recv=49083 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49229 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49458 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49419 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=61440 expected_recv=48960 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49288 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49414 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48809 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49288 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49414 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48809 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49528 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49318 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49580 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49528 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49318 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48960 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=19 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=61440 expected_recv=48960 expected_overflow=0 exact_recv_per_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49031 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48809 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49580 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48960 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=19 calls=1 budget=61440 expected_recv=48960 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49031 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48920 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48809 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49217 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48848 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48920 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49458 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49217 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48848 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=20 calls=1 budget=61440 expected_recv=49458 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49441 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48920 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48736 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49318 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48960 expected_overflow=0 exact_recv_per_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49441 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48920 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48848 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48736 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49318 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=20 calls=1 budget=61440 expected_recv=48960 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49110 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49202 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=21 calls=1 budget=61440 expected_recv=48848 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49065 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49580 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49065 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49580 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49202 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49021 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49202 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49065 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=21 calls=1 budget=61440 expected_recv=49110 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49021 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49441 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49441 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49106 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49126 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49106 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49126 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49106 expected_overflow=0 exact_recv_per_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=22 calls=1 budget=61440 expected_recv=49106 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[48827, 49032, 49008, 49024, 49573, 49408, 49128, 49216] avg=49152.0000 max=49573 max_rank=4 min=48827 min_rank=0 imbalance_ratio=1.008565 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[49340, 49072, 49269, 48925, 49141, 49052, 49110, 49307] avg=49152.0000 max=49340 max_rank=0 min=48925 min_rank=3 imbalance_ratio=1.003825 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[49269, 49149, 49148, 49393, 49149, 49038, 48962, 49108] avg=49152.0000 max=49393 max_rank=3 min=48962 min_rank=6 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49727, 48858, 49082, 48955, 48823] avg=49152.0000 max=49727 max_rank=3 min=48823 min_rank=7 imbalance_ratio=1.011698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[49550, 49096, 49125, 49644, 48858, 49082, 48955, 48906] avg=49152.0000 max=49644 max_rank=3 min=48858 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=83 max_load_reduction_pct=0.001669 moved_to_replicas=83 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[49209, 49110, 49057, 49274, 49026, 49263, 49085, 49192] avg=49152.0000 max=49274 max_rank=3 min=49026 min_rank=4 imbalance_ratio=1.002482 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[48600, 49379, 49167, 49355, 49008, 49690, 48797, 49220] avg=49152.0000 max=49690 max_rank=5 min=48600 min_rank=0 imbalance_ratio=1.010946 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[48646, 49379, 49167, 49355, 49008, 49644, 48797, 49220] avg=49152.0000 max=49644 max_rank=5 min=48646 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=46 max_load_reduction_pct=0.000926 moved_to_replicas=46 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[49290, 49071, 48987, 49299, 49354, 49260, 48925, 49030] avg=49152.0000 max=49354 max_rank=4 min=48925 min_rank=6 imbalance_ratio=1.004110 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[49291, 49118, 49352, 48857, 49031, 49116, 49458, 48993] avg=49152.0000 max=49458 max_rank=6 min=48857 min_rank=3 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[49032, 49537, 48717, 49161, 49040, 49248, 49453, 49028] avg=49152.0000 max=49537 max_rank=1 min=48717 min_rank=2 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=11 tokens_per_ep_rank=[49132, 49071, 49398, 48937, 49297, 48955, 49207, 49219] avg=49152.0000 max=49398 max_rank=2 min=48937 min_rank=3 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=12 tokens_per_ep_rank=[49077, 48827, 49053, 49249, 49292, 48885, 49366, 49467] avg=49152.0000 max=49467 max_rank=7 min=48827 min_rank=1 imbalance_ratio=1.006409 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=13 tokens_per_ep_rank=[48801, 49137, 49172, 49509, 49256, 49042, 49078, 49221] avg=49152.0000 max=49509 max_rank=3 min=48801 min_rank=0 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48876, 49660, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49660 max_rank=2 min=48876 min_rank=1 imbalance_ratio=1.010335 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=14 tokens_per_ep_rank=[49239, 48892, 49644, 49224, 49127, 48900, 48970, 49220] avg=49152.0000 max=49644 max_rank=2 min=48892 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=16 max_load_reduction_pct=0.000322 moved_to_replicas=16 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=15 tokens_per_ep_rank=[49029, 48858, 49132, 49414, 49036, 49439, 49077, 49231] avg=49152.0000 max=49439 max_rank=5 min=48858 min_rank=1 imbalance_ratio=1.005839 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=16 tokens_per_ep_rank=[49348, 49118, 48810, 49318, 49238, 49241, 49227, 48916] avg=49152.0000 max=49348 max_rank=0 min=48810 min_rank=2 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=17 tokens_per_ep_rank=[49254, 49111, 48916, 48901, 49108, 49278, 49298, 49350] avg=49152.0000 max=49350 max_rank=7 min=48901 min_rank=3 imbalance_ratio=1.004028 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=18 tokens_per_ep_rank=[49077, 49393, 49322, 48910, 49084, 48966, 49432, 49032] avg=49152.0000 max=49432 max_rank=6 min=48910 min_rank=3 imbalance_ratio=1.005697 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=19 tokens_per_ep_rank=[48804, 49217, 49231, 49105, 48854, 49358, 49338, 49309] avg=49152.0000 max=49358 max_rank=5 min=48804 min_rank=0 imbalance_ratio=1.004191 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=20 tokens_per_ep_rank=[49083, 49402, 48829, 49368, 49019, 49064, 49438, 49013] avg=49152.0000 max=49438 max_rank=6 min=48829 min_rank=2 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=21 tokens_per_ep_rank=[48960, 49080, 49419, 48605, 49205, 49430, 49288, 49229] avg=49152.0000 max=49430 max_rank=5 min=48605 min_rank=3 imbalance_ratio=1.005656 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=22 tokens_per_ep_rank=[48960, 48920, 49031, 49414, 48809, 49528, 49458, 49096] avg=49152.0000 max=49528 max_rank=5 min=48809 min_rank=4 imbalance_ratio=1.007650 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=23 tokens_per_ep_rank=[49110, 49202, 49318, 49217, 48848, 49205, 49580, 48736] avg=49152.0000 max=49580 max_rank=6 min=48736 min_rank=7 imbalance_ratio=1.008708 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=24 tokens_per_ep_rank=[49126, 49106, 49384, 49065, 49021, 49441, 49051, 49022] avg=49152.0000 max=49441 max_rank=5 min=49021 min_rank=4 imbalance_ratio=1.005880 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=23 total_expected_recv=1131843 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269568 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=23 total_expected_recv=1130061 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49467 required_capacity=1.006409 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=23 total_expected_recv=1129374 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273152 peak_demand=49573 required_capacity=1.008565 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=23 total_expected_recv=1131168 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270592 peak_demand=49644 required_capacity=1.010010 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=23 total_expected_recv=1130489 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269312 peak_demand=49644 required_capacity=1.010010 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=23 total_expected_recv=1131681 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269184 peak_demand=49580 required_capacity=1.008708 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=23 total_expected_recv=1129374 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273152 peak_demand=49573 required_capacity=1.008565 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=23 total_expected_recv=1129454 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272000 peak_demand=49550 required_capacity=1.008097 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=23 total_expected_recv=1131843 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269568 peak_demand=49644 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=23 total_expected_recv=1131168 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270592 peak_demand=49644 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=23 total_expected_recv=1130061 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49467 required_capacity=1.006409 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=23 total_expected_recv=1129898 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271744 peak_demand=49537 required_capacity=1.007833 - [2026-08-26 16:49:22] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 624.1 | throughput per GPU (TFLOP/s/GPU): 204.4 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.983669E+00 | loss scale: 1.0 | grad norm: 3.614 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49273 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=61440 expected_recv=48948 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49350 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] + [2026-08-26 18:01:29] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 625.7 | throughput per GPU (TFLOP/s/GPU): 203.9 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.983669E+00 | loss scale: 1.0 | grad norm: 3.614 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=61440 expected_recv=48948 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49196 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=61440 expected_recv=48848 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49273 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=61440 expected_recv=48948 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49373 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49350 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=61440 expected_recv=48848 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49350 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49273 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=61440 expected_recv=48948 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49196 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49373 expected_overflow=0 exact_recv_per_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=61440 expected_recv=48848 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49373 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49350 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49273 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=61440 expected_recv=49373 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49138 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49138 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49318 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49002 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49138 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49318 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49318 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49002 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49342 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49336 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49342 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49318 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49002 expected_overflow=0 exact_recv_per_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49336 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=61440 expected_recv=49002 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49162 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48787 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49218 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49162 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48787 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49218 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48787 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49336 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49342 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49336 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49353 expected_overflow=0 exact_recv_per_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49342 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49016 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=61440 expected_recv=49353 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49016 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49433 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=61440 expected_recv=48787 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=61440 expected_recv=48885 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49433 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=61440 expected_recv=48885 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49016 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49146 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49016 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49252 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49151 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49433 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49146 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49252 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=61440 expected_recv=48689 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49151 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49252 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=61440 expected_recv=48689 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=61440 expected_recv=48885 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49252 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49417 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49417 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=61440 expected_recv=48885 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=61440 expected_recv=48709 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49331 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49151 expected_overflow=0 exact_recv_per_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49090 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=61440 expected_recv=48709 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=61440 expected_recv=49151 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49331 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=61440 expected_recv=48709 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49032 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=61440 expected_recv=48709 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49331 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49090 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49208 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49041 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49032 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49208 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49041 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49225 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49331 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49417 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49285 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49225 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49417 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49285 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49090 expected_overflow=0 exact_recv_per_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49329 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49009 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49041 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=61440 expected_recv=49090 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=61440 expected_recv=48837 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49329 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49009 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49041 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=61440 expected_recv=48837 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49321 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49225 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48949 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49321 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48950 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49225 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49208 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48949 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48950 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49208 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49329 expected_overflow=0 exact_recv_per_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=61440 expected_recv=49329 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48950 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=61440 expected_recv=48950 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49321 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49199 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49321 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49009 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49199 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49009 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49199 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49149 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=61440 expected_recv=48918 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49148 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49199 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49149 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49307 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=61440 expected_recv=48918 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49148 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49307 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=61440 expected_recv=48771 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=61440 expected_recv=48771 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49295 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49295 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=61440 expected_recv=48928 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49100 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=61440 expected_recv=48771 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49361 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=61440 expected_recv=48928 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49100 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49361 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49295 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49307 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=61440 expected_recv=48771 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49474 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49115 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49361 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49307 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=61440 expected_recv=48918 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49474 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49115 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=61440 expected_recv=48918 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49100 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49035 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49115 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49233 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49100 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49035 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49197 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49115 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49233 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49294 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=61440 expected_recv=48928 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49294 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=61440 expected_recv=48928 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49073 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48939 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49233 expected_overflow=0 exact_recv_per_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49197 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49073 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48939 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49294 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=61440 expected_recv=49233 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49505 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48939 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48975 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49505 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48939 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48975 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49294 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49505 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48975 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49239 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=61440 expected_recv=48969 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48975 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49197 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49239 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49194 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=61440 expected_recv=48969 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49413 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49197 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=10 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49194 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49413 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49194 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=10 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49239 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49194 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49446 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=61440 expected_recv=48792 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49239 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49446 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49011 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49466 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=11 calls=1 budget=61440 expected_recv=48792 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49446 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49241 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49413 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49011 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49466 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49446 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=61440 expected_recv=48969 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49413 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49144 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49241 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=11 calls=1 budget=61440 expected_recv=48969 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49152 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=61440 expected_recv=48932 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49144 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49011 expected_overflow=0 exact_recv_per_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49144 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=61440 expected_recv=48932 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=11 calls=1 budget=61440 expected_recv=49011 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49466 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49018 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49466 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49144 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49018 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49442 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49018 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49240 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49183 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49018 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49240 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=61440 expected_recv=48932 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49183 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49442 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49241 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=12 calls=1 budget=61440 expected_recv=48932 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=61440 expected_recv=48848 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49241 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=12 calls=1 budget=61440 expected_recv=48848 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49183 expected_overflow=0 exact_recv_per_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49240 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=12 calls=1 budget=61440 expected_recv=49183 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49017 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49240 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49017 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49017 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49158 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49300 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49176 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49300 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=61440 expected_recv=48802 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49158 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49176 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=61440 expected_recv=48802 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49017 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49158 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49135 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=13 calls=1 budget=61440 expected_recv=48802 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49442 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49442 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49300 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=13 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49300 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=61440 expected_recv=48981 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49135 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49400 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49135 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=14 calls=1 budget=61440 expected_recv=48981 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=61440 expected_recv=48908 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49048 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49135 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49048 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49301 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49176 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49400 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49301 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49176 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=61440 expected_recv=48908 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=14 calls=1 budget=61440 expected_recv=48908 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49316 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49301 expected_overflow=0 exact_recv_per_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49406 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49316 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=14 calls=1 budget=61440 expected_recv=49301 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49048 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=61440 expected_recv=48968 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48864 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49048 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49406 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49018 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49406 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=61440 expected_recv=48968 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48864 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48827 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49316 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49400 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49174 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49406 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48827 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49316 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49029 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49018 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49400 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=15 calls=1 budget=61440 expected_recv=49174 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49029 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49436 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48827 expected_overflow=0 exact_recv_per_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49436 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=15 calls=1 budget=61440 expected_recv=48827 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=61440 expected_recv=48968 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=16 calls=1 budget=61440 expected_recv=48968 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49218 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=61440 expected_recv=48810 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49388 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49295 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49218 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49407 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=61440 expected_recv=48810 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48971 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49388 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=61440 expected_recv=48810 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49295 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=16 calls=1 budget=61440 expected_recv=48810 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49330 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49436 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49018 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49330 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49436 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49407 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49018 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48971 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49407 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48971 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48877 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48773 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49330 expected_overflow=0 exact_recv_per_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49029 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48877 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=16 calls=1 budget=61440 expected_recv=49330 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49295 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49295 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48773 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49300 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49537 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48773 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49537 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=61440 expected_recv=48876 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49300 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49300 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49029 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=61440 expected_recv=48876 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49300 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49417 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48877 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49388 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49417 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49029 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=17 calls=1 budget=61440 expected_recv=48877 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=61440 expected_recv=48876 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49388 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49029 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=17 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=61440 expected_recv=48689 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=18 calls=1 budget=61440 expected_recv=48876 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49109 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=18 calls=1 budget=61440 expected_recv=48689 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49183 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49279 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49109 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49288 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49183 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49279 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49473 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49288 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49473 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49109 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49537 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49183 expected_overflow=0 exact_recv_per_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49279 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49109 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49537 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=18 calls=1 budget=61440 expected_recv=49183 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49378 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49279 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49378 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48993 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49042 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=61440 expected_recv=48990 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48798 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49312 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48993 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49042 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49312 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49042 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=61440 expected_recv=48990 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48798 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49106 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49312 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49288 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49106 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49312 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49262 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49288 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=61440 expected_recv=48990 expected_overflow=0 exact_recv_per_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49045 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48993 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49262 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=19 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=19 calls=1 budget=61440 expected_recv=48990 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48993 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49262 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49045 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49188 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48924 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49314 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49346 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49188 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49512 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48924 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49314 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49346 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48798 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49512 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49188 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49512 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49188 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=61440 expected_recv=48440 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=20 calls=1 budget=61440 expected_recv=48798 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=61440 expected_recv=48440 actual_valid_recv=48896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49398 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49490 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49398 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49490 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49284 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49346 expected_overflow=0 exact_recv_per_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=61440 expected_recv=48440 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=20 calls=1 budget=61440 expected_recv=49346 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=21 calls=1 budget=61440 expected_recv=48440 actual_valid_recv=48896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49398 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49284 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48991 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=48896 slack_slots=12544 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49284 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49398 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49284 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49314 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49314 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49045 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49490 expected_overflow=0 exact_recv_per_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48991 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49045 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48991 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=21 calls=1 budget=61440 expected_recv=49490 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49126 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49315 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49126 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49315 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=22 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49126 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49313 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49126 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49313 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49315 expected_overflow=0 exact_recv_per_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=22 calls=1 budget=61440 expected_recv=49315 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[49373, 49196, 49350, 48848, 48948, 49137, 49273, 49091] avg=49152.0000 max=49373 max_rank=0 min=48848 min_rank=3 imbalance_ratio=1.004496 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[49002, 49292, 49128, 49318, 49138, 49214, 49091, 49033] avg=49152.0000 max=49318 max_rank=3 min=49002 min_rank=0 imbalance_ratio=1.003377 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[49353, 49162, 49336, 49218, 48927, 49091, 49342, 48787] avg=49152.0000 max=49353 max_rank=0 min=48787 min_rank=7 imbalance_ratio=1.004089 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49645, 48688] avg=49152.0000 max=49645 max_rank=6 min=48688 min_rank=7 imbalance_ratio=1.010030 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[49151, 49146, 48885, 49252, 49433, 49016, 49644, 48689] avg=49152.0000 max=49644 max_rank=6 min=48689 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=1 max_load_reduction_pct=0.000020 moved_to_replicas=1 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[49090, 48709, 49417, 49331, 49214, 49246, 49177, 49032] avg=49152.0000 max=49417 max_rank=2 min=48709 min_rank=1 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[49329, 49077, 49225, 49214, 49041, 49285, 49208, 48837] avg=49152.0000 max=49329 max_rank=0 min=48837 min_rank=7 imbalance_ratio=1.003601 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49656, 48937, 49009, 49096] avg=49152.0000 max=49656 max_rank=4 min=48937 min_rank=5 imbalance_ratio=1.010254 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[49070, 48950, 49321, 49177, 49644, 48949, 49009, 49096] avg=49152.0000 max=49644 max_rank=4 min=48949 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000242 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[49001, 49199, 49111, 49149, 49137, 49257, 49214, 49148] avg=49152.0000 max=49257 max_rank=5 min=49001 min_rank=0 imbalance_ratio=1.002136 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[49116, 48771, 49307, 49295, 49125, 49323, 48918, 49361] avg=49152.0000 max=49361 max_rank=7 min=48771 min_rank=1 imbalance_ratio=1.004252 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=11 tokens_per_ep_rank=[49073, 49115, 49258, 49035, 49100, 49474, 48928, 49233] avg=49152.0000 max=49474 max_rank=5 min=48928 min_rank=6 imbalance_ratio=1.006551 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=12 tokens_per_ep_rank=[48967, 49505, 49260, 48975, 48939, 49294, 49197, 49079] avg=49152.0000 max=49505 max_rank=1 min=48939 min_rank=4 imbalance_ratio=1.007182 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=13 tokens_per_ep_rank=[49152, 48792, 49413, 49446, 49194, 49239, 48969, 49011] avg=49152.0000 max=49446 max_rank=3 min=48792 min_rank=1 imbalance_ratio=1.005981 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=14 tokens_per_ep_rank=[48848, 49018, 48932, 49384, 49144, 49466, 49241, 49183] avg=49152.0000 max=49466 max_rank=5 min=48848 min_rank=0 imbalance_ratio=1.006388 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=15 tokens_per_ep_rank=[49226, 49017, 49116, 49173, 49200, 49240, 49442, 48802] avg=49152.0000 max=49442 max_rank=6 min=48802 min_rank=7 imbalance_ratio=1.005900 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=16 tokens_per_ep_rank=[48908, 49135, 49176, 48981, 49300, 49158, 49257, 49301] avg=49152.0000 max=49301 max_rank=7 min=48908 min_rank=0 imbalance_ratio=1.003031 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=17 tokens_per_ep_rank=[49174, 48864, 49316, 49406, 49181, 49048, 49400, 48827] avg=49152.0000 max=49406 max_rank=3 min=48827 min_rank=7 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=18 tokens_per_ep_rank=[49407, 48810, 49436, 49218, 49029, 48968, 49018, 49330] avg=49152.0000 max=49436 max_rank=2 min=48810 min_rank=1 imbalance_ratio=1.005778 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=19 tokens_per_ep_rank=[49195, 49300, 48877, 48773, 48971, 49295, 49388, 49417] avg=49152.0000 max=49417 max_rank=7 min=48773 min_rank=3 imbalance_ratio=1.005391 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=20 tokens_per_ep_rank=[49183, 49292, 49137, 48689, 48876, 49029, 49537, 49473] avg=49152.0000 max=49537 max_rank=6 min=48689 min_rank=3 imbalance_ratio=1.007833 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=21 tokens_per_ep_rank=[48990, 49042, 49288, 49378, 49279, 49109, 49024, 49106] avg=49152.0000 max=49378 max_rank=3 min=48990 min_rank=0 imbalance_ratio=1.004598 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=22 tokens_per_ep_rank=[49346, 48924, 48798, 49262, 48993, 49312, 49069, 49512] avg=49152.0000 max=49512 max_rank=7 min=48798 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=23 total_expected_recv=1131448 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269440 peak_demand=49644 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=23 total_expected_recv=1130060 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271616 peak_demand=49644 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=23 total_expected_recv=1131368 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270080 peak_demand=49436 required_capacity=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=23 tokens_per_ep_rank=[49490, 49398, 49314, 48440, 49057, 49188, 49045, 49284] avg=49152.0000 max=49490 max_rank=0 min=48440 min_rank=3 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=24 tokens_per_ep_rank=[49313, 49126, 48967, 48991, 49190, 49257, 49057, 49315] avg=49152.0000 max=49315 max_rank=7 min=48967 min_rank=2 imbalance_ratio=1.003316 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=23 total_expected_recv=1131595 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269696 peak_demand=49474 required_capacity=1.006551 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=23 total_expected_recv=1130757 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270976 peak_demand=49490 required_capacity=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=23 total_expected_recv=1130060 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271616 peak_demand=49644 required_capacity=1.010010 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=23 total_expected_recv=1129953 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271616 peak_demand=49446 required_capacity=1.005981 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=23 total_expected_recv=1128840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272000 peak_demand=49505 required_capacity=1.007182 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=23 total_expected_recv=1129947 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49512 required_capacity=1.007324 - [2026-08-26 16:49:23] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 624.8 | throughput per GPU (TFLOP/s/GPU): 204.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.792192E+00 | loss scale: 1.0 | grad norm: 5.623 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=23 total_expected_recv=1131368 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270080 peak_demand=49436 required_capacity=1.005778 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=23 total_expected_recv=1128840 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272000 peak_demand=49505 required_capacity=1.007182 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=23 total_expected_recv=1130757 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270976 peak_demand=49490 required_capacity=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=23 total_expected_recv=1131448 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269440 peak_demand=49644 required_capacity=1.010010 + [2026-08-26 18:01:29] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 625.1 | throughput per GPU (TFLOP/s/GPU): 204.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.792192E+00 | loss scale: 1.0 | grad norm: 5.623 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48943 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49240 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49036 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48943 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48867 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48943 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49240 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49036 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48943 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48867 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=61440 expected_recv=48867 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=61440 expected_recv=48831 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49333 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49282 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=61440 expected_recv=48831 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=61440 expected_recv=48831 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=61440 expected_recv=48945 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49333 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49282 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=61440 expected_recv=48831 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=61440 expected_recv=48945 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49357 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=61440 expected_recv=48945 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49282 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49282 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49357 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49256 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=61440 expected_recv=48825 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49357 expected_overflow=0 exact_recv_per_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49233 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=61440 expected_recv=48825 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49256 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=61440 expected_recv=48825 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49335 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=61440 expected_recv=49357 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49233 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49006 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=61440 expected_recv=48825 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49335 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49335 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49256 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49006 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=61440 expected_recv=48988 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49327 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49335 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49006 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49256 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49006 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48707 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49327 expected_overflow=0 exact_recv_per_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48707 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49032 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=61440 expected_recv=49327 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49074 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49121 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49032 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49074 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49032 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49399 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49121 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49172 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48707 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49074 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49540 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49399 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49121 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49172 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=61440 expected_recv=48707 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49172 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49399 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49540 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49172 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49218 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49399 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49018 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49218 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49540 expected_overflow=0 exact_recv_per_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49018 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=61440 expected_recv=49540 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=61440 expected_recv=48791 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49018 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49493 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49192 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=61440 expected_recv=48791 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49407 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49218 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49493 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49192 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=61440 expected_recv=48791 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=61440 expected_recv=48943 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49493 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49192 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49407 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49218 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49493 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49203 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=61440 expected_recv=49192 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=61440 expected_recv=48943 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49203 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=61440 expected_recv=48943 expected_overflow=0 exact_recv_per_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49272 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=61440 expected_recv=48943 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=61440 expected_recv=48906 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49272 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=61440 expected_recv=48906 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49114 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49114 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49025 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49461 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49203 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49025 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49025 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49160 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49114 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49461 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49025 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49160 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=61440 expected_recv=48916 expected_overflow=0 exact_recv_per_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=61440 expected_recv=49203 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=61440 expected_recv=48916 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49553 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=61440 expected_recv=48829 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=61440 expected_recv=48888 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49553 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=61440 expected_recv=48829 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=61440 expected_recv=48888 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=61440 expected_recv=48888 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49553 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49160 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49082 expected_overflow=0 exact_recv_per_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=61440 expected_recv=48888 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=61440 expected_recv=48978 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49553 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49343 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49160 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49082 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=61440 expected_recv=49082 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=61440 expected_recv=48978 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49343 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49343 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49515 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49343 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49515 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49184 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49023 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49184 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49023 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49023 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49515 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=61440 expected_recv=48829 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49174 expected_overflow=0 exact_recv_per_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49023 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49290 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49515 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=61440 expected_recv=48829 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=61440 expected_recv=49174 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49290 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49280 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=61440 expected_recv=48820 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49333 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49280 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=61440 expected_recv=48820 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49333 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49178 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=61440 expected_recv=48820 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49333 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49178 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49280 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49184 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=61440 expected_recv=48820 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=61440 expected_recv=48935 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49109 expected_overflow=0 exact_recv_per_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49019 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49280 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49184 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=61440 expected_recv=48935 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49094 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=61440 expected_recv=49109 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49019 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49094 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=61440 expected_recv=48935 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49241 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49241 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49210 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49266 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=61440 expected_recv=48935 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49210 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49266 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49266 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49094 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49210 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49266 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49241 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49094 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49241 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49444 expected_overflow=0 exact_recv_per_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49367 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=61440 expected_recv=48900 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49266 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=61440 expected_recv=49444 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49367 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=61440 expected_recv=48900 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49078 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49242 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49242 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49078 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49107 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49230 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=61440 expected_recv=48896 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49178 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49266 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=61440 expected_recv=48900 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49107 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49230 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49242 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=10 calls=1 budget=61440 expected_recv=48896 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49107 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=10 calls=1 budget=61440 expected_recv=48900 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49360 expected_overflow=0 exact_recv_per_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49178 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49035 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49266 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49107 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49242 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=10 calls=1 budget=61440 expected_recv=49360 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49110 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49035 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49178 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49375 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49375 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49178 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=61440 expected_recv=48970 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49172 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49316 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49035 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=61440 expected_recv=48970 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49172 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49316 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49230 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49375 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49035 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49146 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=61440 expected_recv=48970 expected_overflow=0 exact_recv_per_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49375 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49357 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49055 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49230 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=11 calls=1 budget=61440 expected_recv=49146 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=11 calls=1 budget=61440 expected_recv=48970 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49357 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49055 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49316 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49398 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49398 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49316 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49055 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=61440 expected_recv=48933 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49479 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49055 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=12 calls=1 budget=61440 expected_recv=48933 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49479 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49055 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49398 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49115 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49055 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49186 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49055 expected_overflow=0 exact_recv_per_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=12 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49398 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49115 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49186 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=12 calls=1 budget=61440 expected_recv=49055 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49479 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49387 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49479 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49387 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=61440 expected_recv=48923 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49406 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49088 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=13 calls=1 budget=61440 expected_recv=48923 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49406 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49088 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49003 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49267 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=61440 expected_recv=48791 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49146 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49088 expected_overflow=0 exact_recv_per_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49003 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49267 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=61440 expected_recv=48791 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49146 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=13 calls=1 budget=61440 expected_recv=49088 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49375 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49406 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49406 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=61440 expected_recv=48918 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=61440 expected_recv=48942 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49375 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=14 calls=1 budget=61440 expected_recv=48918 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=61440 expected_recv=48942 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49074 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49387 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=61440 expected_recv=48791 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49375 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49119 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49546 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49089 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49387 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49074 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=14 calls=1 budget=61440 expected_recv=48791 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49375 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49119 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49546 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49074 expected_overflow=0 exact_recv_per_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49089 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49061 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=14 calls=1 budget=61440 expected_recv=49074 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=61440 expected_recv=48942 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49061 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=61440 expected_recv=48775 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49083 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49288 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=15 calls=1 budget=61440 expected_recv=48942 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=15 calls=1 budget=61440 expected_recv=48775 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49083 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49288 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49307 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49142 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49491 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49142 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49307 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49262 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49491 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49546 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49288 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49262 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49546 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49491 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49307 expected_overflow=0 exact_recv_per_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49288 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=61440 expected_recv=48988 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49491 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=15 calls=1 budget=61440 expected_recv=49307 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49083 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49097 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=61440 expected_recv=48990 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49083 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49097 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=61440 expected_recv=48990 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49061 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49146 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49434 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49399 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49142 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=61440 expected_recv=48918 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49061 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49146 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49434 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=61440 expected_recv=48990 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49142 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=61440 expected_recv=48918 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49399 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49434 expected_overflow=0 exact_recv_per_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49039 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=16 calls=1 budget=61440 expected_recv=48990 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49399 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=16 calls=1 budget=61440 expected_recv=49434 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49039 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=61440 expected_recv=48583 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=16 calls=1 budget=61440 expected_recv=48918 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49450 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=17 calls=1 budget=61440 expected_recv=48583 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49281 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49450 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49281 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=61440 expected_recv=48988 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49434 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=17 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49146 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49381 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49434 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49146 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49281 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49434 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49381 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49272 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49281 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49434 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49272 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=17 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49085 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=61440 expected_recv=48842 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49085 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=61440 expected_recv=48872 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=61440 expected_recv=48842 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49450 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=61440 expected_recv=48872 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49039 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49039 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49450 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49042 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49042 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=61440 expected_recv=48952 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=61440 expected_recv=48872 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=61440 expected_recv=48952 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49056 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49042 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=18 calls=1 budget=61440 expected_recv=48872 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=18 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49056 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49174 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=61440 expected_recv=48842 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49174 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49272 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=61440 expected_recv=48897 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49191 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49272 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49090 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=61440 expected_recv=48897 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49191 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=19 calls=1 budget=61440 expected_recv=48842 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49242 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49090 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49434 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49242 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=61440 expected_recv=48659 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49434 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49090 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49463 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=61440 expected_recv=48659 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49267 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49267 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=61440 expected_recv=48952 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49288 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49228 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49053 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=20 calls=1 budget=61440 expected_recv=48952 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49288 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49228 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49174 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49090 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49191 expected_overflow=0 exact_recv_per_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49463 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49434 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49174 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=19 calls=1 budget=61440 expected_recv=49191 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49434 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=61440 expected_recv=48897 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49088 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=20 calls=1 budget=61440 expected_recv=48897 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49088 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49053 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49219 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49267 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49053 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49219 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49249 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49196 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49088 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49228 expected_overflow=0 exact_recv_per_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49267 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49249 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49196 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49387 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=20 calls=1 budget=61440 expected_recv=49228 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49387 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49160 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49463 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=61440 expected_recv=48659 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49288 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=61440 expected_recv=48992 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=21 calls=1 budget=61440 expected_recv=48659 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49288 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=61440 expected_recv=48992 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=61440 expected_recv=48831 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49160 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49463 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49387 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=22 calls=1 budget=61440 expected_recv=48831 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=61440 expected_recv=48992 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=61440 expected_recv=48936 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49387 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=21 calls=1 budget=61440 expected_recv=48992 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=22 calls=1 budget=61440 expected_recv=48936 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=21 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49160 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49196 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49473 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49160 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49196 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49473 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49289 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49289 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49219 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49112 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49219 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49112 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49473 expected_overflow=0 exact_recv_per_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=22 calls=1 budget=61440 expected_recv=49473 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49676, 49240, 48943, 49036, 48835] avg=49152.0000 max=49676 max_rank=3 min=48835 min_rank=7 imbalance_ratio=1.010661 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[49263, 49204, 49019, 49644, 49240, 48943, 49036, 48867] avg=49152.0000 max=49644 max_rank=3 min=48867 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32 max_load_reduction_pct=0.000644 moved_to_replicas=32 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[49357, 49282, 49319, 48945, 49333, 49077, 48831, 49072] avg=49152.0000 max=49357 max_rank=0 min=48831 min_rank=6 imbalance_ratio=1.004171 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[49327, 49006, 49246, 49335, 49233, 49256, 48825, 48988] avg=49152.0000 max=49335 max_rank=3 min=48825 min_rank=6 imbalance_ratio=1.003723 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[49540, 49399, 49172, 49121, 49032, 48707, 49074, 49171] avg=49152.0000 max=49540 max_rank=0 min=48707 min_rank=5 imbalance_ratio=1.007894 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[48943, 49192, 49493, 48791, 49154, 49218, 49018, 49407] avg=49152.0000 max=49493 max_rank=2 min=48791 min_rank=3 imbalance_ratio=1.006938 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[48916, 49025, 49114, 49319, 49272, 49203, 48906, 49461] avg=49152.0000 max=49461 max_rank=7 min=48906 min_rank=6 imbalance_ratio=1.006287 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[49082, 48888, 49057, 49079, 49226, 49160, 49171, 49553] avg=49152.0000 max=49553 max_rank=7 min=48888 min_rank=1 imbalance_ratio=1.008158 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[49174, 49023, 49132, 49515, 48978, 48829, 49343, 49222] avg=49152.0000 max=49515 max_rank=3 min=48829 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[49109, 49178, 49333, 49280, 49290, 49184, 49022, 48820] avg=49152.0000 max=49333 max_rank=2 min=48820 min_rank=7 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=11 tokens_per_ep_rank=[49444, 49241, 49094, 49266, 49019, 49007, 48935, 49210] avg=49152.0000 max=49444 max_rank=0 min=48935 min_rank=6 imbalance_ratio=1.005941 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=12 tokens_per_ep_rank=[49360, 49242, 48900, 49107, 49367, 49266, 49078, 48896] avg=49152.0000 max=49367 max_rank=4 min=48896 min_rank=7 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=13 tokens_per_ep_rank=[49146, 49375, 49035, 48970, 49110, 49230, 49178, 49172] avg=49152.0000 max=49375 max_rank=1 min=48970 min_rank=3 imbalance_ratio=1.004537 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=14 tokens_per_ep_rank=[49115, 49398, 49055, 49055, 49357, 48987, 49316, 48933] avg=49152.0000 max=49398 max_rank=1 min=48933 min_rank=7 imbalance_ratio=1.005005 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=15 tokens_per_ep_rank=[49267, 49066, 49204, 49088, 49186, 49003, 49479, 48923] avg=49152.0000 max=49479 max_rank=6 min=48923 min_rank=7 imbalance_ratio=1.006653 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=16 tokens_per_ep_rank=[49119, 49375, 48791, 49074, 49146, 49387, 49406, 48918] avg=49152.0000 max=49406 max_rank=6 min=48791 min_rank=2 imbalance_ratio=1.005168 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=17 tokens_per_ep_rank=[49307, 49288, 49546, 49262, 49089, 49007, 48942, 48775] avg=49152.0000 max=49546 max_rank=2 min=48775 min_rank=7 imbalance_ratio=1.008016 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=18 tokens_per_ep_rank=[49434, 48990, 49142, 48918, 49491, 49061, 49083, 49097] avg=49152.0000 max=49491 max_rank=4 min=48918 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=19 tokens_per_ep_rank=[49251, 49281, 49146, 49381, 49399, 48988, 49187, 48583] avg=49152.0000 max=49399 max_rank=4 min=48583 min_rank=7 imbalance_ratio=1.005025 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=20 tokens_per_ep_rank=[49072, 48872, 49222, 49042, 49434, 49039, 49450, 49085] avg=49152.0000 max=49450 max_rank=6 min=48872 min_rank=1 imbalance_ratio=1.006063 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=21 tokens_per_ep_rank=[49191, 49174, 49258, 49242, 49181, 49272, 48842, 49056] avg=49152.0000 max=49272 max_rank=5 min=48842 min_rank=6 imbalance_ratio=1.002441 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=22 tokens_per_ep_rank=[49228, 49267, 48897, 49088, 49090, 48952, 49260, 49434] avg=49152.0000 max=49434 max_rank=7 min=48897 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=23 tokens_per_ep_rank=[49125, 49387, 49288, 48992, 49053, 49463, 48659, 49249] avg=49152.0000 max=49463 max_rank=5 min=48659 min_rank=6 imbalance_ratio=1.006327 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=23 total_expected_recv=1129237 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272512 peak_demand=49479 required_capacity=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=24 tokens_per_ep_rank=[49112, 49289, 49160, 49473, 48831, 49219, 49196, 48936] avg=49152.0000 max=49473 max_rank=3 min=48831 min_rank=4 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=23 total_expected_recv=1129458 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271744 peak_demand=49463 required_capacity=1.006327 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=23 total_expected_recv=1131511 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269824 peak_demand=49491 required_capacity=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=23 total_expected_recv=1128828 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273280 peak_demand=49553 required_capacity=1.008158 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=23 total_expected_recv=1130623 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49546 required_capacity=1.008016 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=23 total_expected_recv=1129237 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272512 peak_demand=49479 required_capacity=1.006653 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=23 total_expected_recv=1130987 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49644 required_capacity=1.010010 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=23 total_expected_recv=1131882 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268800 peak_demand=49540 required_capacity=1.007894 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=23 total_expected_recv=1129458 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271744 peak_demand=49463 required_capacity=1.006327 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=23 total_expected_recv=1128828 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273280 peak_demand=49553 required_capacity=1.008158 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=23 total_expected_recv=1131442 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270080 peak_demand=49399 required_capacity=1.005025 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=23 total_expected_recv=1130987 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49644 required_capacity=1.010010 - [2026-08-26 16:49:24] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 625.7 | throughput per GPU (TFLOP/s/GPU): 203.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.617704E+00 | loss scale: 1.0 | grad norm: 312.445 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49301 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] + [2026-08-26 18:01:30] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 626.0 | throughput per GPU (TFLOP/s/GPU): 203.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.617704E+00 | loss scale: 1.0 | grad norm: 312.445 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48860 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49094 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49243 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49324 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49243 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48850 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49301 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49094 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49301 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48860 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49094 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49243 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49324 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49322 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49243 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=61440 expected_recv=48850 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49094 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49301 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49163 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49081 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49050 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=61440 expected_recv=49322 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49163 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49260 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49050 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49045 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49081 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49050 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49260 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49081 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49050 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49045 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49081 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49146 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49228 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49146 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49346 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=61440 expected_recv=48945 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49228 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49391 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49199 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=61440 expected_recv=48818 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49346 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49143 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=61440 expected_recv=48945 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=61440 expected_recv=48945 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=61440 expected_recv=48818 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49391 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49199 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49346 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=61440 expected_recv=48945 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49146 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49143 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=61440 expected_recv=48818 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49143 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49346 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49146 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49143 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48925 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49554 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48925 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49554 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49199 expected_overflow=0 exact_recv_per_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49040 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49097 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=61440 expected_recv=49199 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48768 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49418 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49040 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49097 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49097 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48768 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49418 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48925 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49209 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49209 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49360 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49360 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49097 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=61440 expected_recv=48925 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49209 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49418 expected_overflow=0 exact_recv_per_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49459 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49040 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48835 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48985 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=61440 expected_recv=49418 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49459 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49040 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48835 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48985 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49040 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49276 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49276 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49360 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49040 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49175 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49360 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49175 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48835 expected_overflow=0 exact_recv_per_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49220 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=61440 expected_recv=48691 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49381 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=61440 expected_recv=48835 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49220 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=61440 expected_recv=48691 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49381 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48977 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48977 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49276 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49210 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49381 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49276 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49210 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48719 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49381 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49300 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49020 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49312 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49220 expected_overflow=0 exact_recv_per_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49510 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48719 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49300 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49020 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=61440 expected_recv=48975 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49312 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=61440 expected_recv=49220 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49510 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=61440 expected_recv=48975 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48719 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48719 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49393 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48977 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49312 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49393 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49196 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49389 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=61440 expected_recv=48977 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49314 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=61440 expected_recv=48847 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49196 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49389 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49314 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49312 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49022 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=61440 expected_recv=48847 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49020 expected_overflow=0 exact_recv_per_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49314 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49022 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=61440 expected_recv=49020 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49389 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49196 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49389 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49196 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49267 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=61440 expected_recv=48698 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=61440 expected_recv=48975 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=61440 expected_recv=48975 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49156 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=61440 expected_recv=48764 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49267 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=61440 expected_recv=48698 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49156 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49374 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=61440 expected_recv=48764 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49507 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49413 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49374 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=61440 expected_recv=48847 expected_overflow=0 exact_recv_per_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49156 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49507 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49413 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=61440 expected_recv=48847 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=61440 expected_recv=48698 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=61440 expected_recv=48698 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49267 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49205 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49267 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49205 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=61440 expected_recv=48803 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49374 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49342 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=61440 expected_recv=48998 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=61440 expected_recv=48803 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49034 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49289 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=61440 expected_recv=48861 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=61440 expected_recv=48764 expected_overflow=0 exact_recv_per_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=61440 expected_recv=49374 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49342 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=61440 expected_recv=48998 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49034 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49289 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=61440 expected_recv=48998 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=61440 expected_recv=48764 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=61440 expected_recv=48861 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=61440 expected_recv=48998 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=61440 expected_recv=48803 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49295 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=61440 expected_recv=48803 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49295 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49162 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49352 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49245 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49413 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49034 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49162 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49277 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49352 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=61440 expected_recv=48980 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49413 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49034 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49245 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49277 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49132 expected_overflow=0 exact_recv_per_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49245 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=10 calls=1 budget=61440 expected_recv=48980 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=61440 expected_recv=49132 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49245 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49352 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49408 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49352 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49408 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49160 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48954 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=61440 expected_recv=48861 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49277 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48969 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49160 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49311 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48999 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48954 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49156 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49247 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=10 calls=1 budget=61440 expected_recv=48861 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48969 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49311 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48999 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49156 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49277 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49311 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49162 expected_overflow=0 exact_recv_per_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49247 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49269 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48954 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49311 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=10 calls=1 budget=61440 expected_recv=49162 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49269 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48954 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49298 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48845 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48941 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49298 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48999 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48845 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48941 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48941 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=11 calls=1 budget=61440 expected_recv=48999 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49160 expected_overflow=0 exact_recv_per_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49125 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49298 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=12 calls=1 budget=61440 expected_recv=48941 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=11 calls=1 budget=61440 expected_recv=49160 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49125 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49298 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49011 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49011 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49120 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49569 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49156 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49569 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49107 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49156 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49107 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49329 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=61440 expected_recv=48942 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=61440 expected_recv=48889 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=61440 expected_recv=48796 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49107 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49120 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49569 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49120 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49329 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=61440 expected_recv=48942 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=61440 expected_recv=48889 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=12 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=14 calls=1 budget=61440 expected_recv=48796 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49107 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49120 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49427 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=61440 expected_recv=48889 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49427 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49089 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=13 calls=1 budget=61440 expected_recv=48889 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49089 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49010 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49188 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49427 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49242 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49011 expected_overflow=0 exact_recv_per_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49234 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49010 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49188 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49010 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49427 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49089 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=13 calls=1 budget=61440 expected_recv=49011 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49234 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49242 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49010 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49089 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48998 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48998 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49488 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49488 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49456 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49160 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48895 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=61440 expected_recv=48942 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49456 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49160 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48895 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=14 calls=1 budget=61440 expected_recv=48942 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48895 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49456 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=14 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49456 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=61440 expected_recv=48913 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49160 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=16 calls=1 budget=61440 expected_recv=48913 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49367 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49368 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49242 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49242 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49367 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49368 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49160 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49242 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49033 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=15 calls=1 budget=61440 expected_recv=49242 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49242 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49033 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48998 expected_overflow=0 exact_recv_per_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49368 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49368 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=15 calls=1 budget=61440 expected_recv=48998 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=61440 expected_recv=48967 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=17 calls=1 budget=61440 expected_recv=48967 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=61440 expected_recv=48988 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49005 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49549 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=61440 expected_recv=48703 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=61440 expected_recv=48957 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49005 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49549 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=61440 expected_recv=48703 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49005 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=16 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=61440 expected_recv=48957 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49005 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=61440 expected_recv=48703 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=16 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49403 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=17 calls=1 budget=61440 expected_recv=48703 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49403 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49000 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49549 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49462 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49082 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49000 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49549 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49367 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49147 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49462 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49082 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49462 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49377 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49367 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49147 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49017 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49462 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49377 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=17 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49075 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49017 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49075 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49082 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49218 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49218 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49433 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=18 calls=1 budget=61440 expected_recv=49082 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=61440 expected_recv=48988 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49433 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49017 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=18 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=61440 expected_recv=48957 expected_overflow=0 exact_recv_per_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48949 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49017 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49433 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=18 calls=1 budget=61440 expected_recv=48957 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49462 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48949 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49433 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49462 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49041 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49490 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49041 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48958 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49000 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48989 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49490 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49238 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48958 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49490 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48949 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49000 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49069 expected_overflow=0 exact_recv_per_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48989 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49238 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49325 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49490 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48949 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49017 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49325 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49264 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=19 calls=1 budget=61440 expected_recv=49069 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49017 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49264 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48958 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=20 calls=1 budget=61440 expected_recv=48958 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49218 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=61440 expected_recv=48776 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49115 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=61440 expected_recv=48928 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49325 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49218 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=61440 expected_recv=48776 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49181 expected_overflow=0 exact_recv_per_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49410 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49115 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=61440 expected_recv=48928 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49325 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=61440 expected_recv=48776 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=20 calls=1 budget=61440 expected_recv=49181 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49410 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=61440 expected_recv=48928 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=21 calls=1 budget=61440 expected_recv=48928 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49460 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49041 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=61440 expected_recv=48932 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=21 calls=1 budget=61440 expected_recv=48776 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49410 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=61440 expected_recv=48932 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49410 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49094 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49460 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49238 expected_overflow=0 exact_recv_per_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49041 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=22 calls=1 budget=61440 expected_recv=48932 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=61440 expected_recv=48899 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49094 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49460 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=21 calls=1 budget=61440 expected_recv=49238 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=22 calls=1 budget=61440 expected_recv=48899 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49460 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49094 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49094 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49195 expected_overflow=0 exact_recv_per_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=22 calls=1 budget=61440 expected_recv=49195 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[49222, 49324, 49322, 49094, 48850, 48860, 49301, 49243] avg=49152.0000 max=49324 max_rank=1 min=48850 min_rank=4 imbalance_ratio=1.003499 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[49222, 49260, 49222, 49081, 49045, 49163, 49173, 49050] avg=49152.0000 max=49260 max_rank=1 min=49045 min_rank=4 imbalance_ratio=1.002197 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[49143, 49391, 49199, 49346, 48818, 49228, 49146, 48945] avg=49152.0000 max=49391 max_rank=1 min=48818 min_rank=4 imbalance_ratio=1.004862 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[49209, 48768, 49418, 49205, 49097, 49554, 48925, 49040] avg=49152.0000 max=49554 max_rank=5 min=48768 min_rank=1 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[49377, 48985, 48835, 49024, 49040, 49136, 49360, 49459] avg=49152.0000 max=49459 max_rank=7 min=48835 min_rank=2 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[49381, 48691, 49220, 49181, 49070, 49175, 49276, 49222] avg=49152.0000 max=49381 max_rank=0 min=48691 min_rank=1 imbalance_ratio=1.004659 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[49312, 49510, 49020, 49168, 48719, 49210, 48977, 49300] avg=49152.0000 max=49510 max_rank=1 min=48719 min_rank=4 imbalance_ratio=1.007284 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[49080, 49022, 48847, 49196, 49389, 49393, 48975, 49314] avg=49152.0000 max=49393 max_rank=5 min=48847 min_rank=2 imbalance_ratio=1.004903 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[49374, 49507, 48764, 49267, 48698, 49263, 49187, 49156] avg=49152.0000 max=49507 max_rank=1 min=48698 min_rank=4 imbalance_ratio=1.007222 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=11 tokens_per_ep_rank=[49034, 49289, 49132, 48803, 48998, 49205, 49413, 49342] avg=49152.0000 max=49413 max_rank=6 min=48803 min_rank=3 imbalance_ratio=1.005310 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=12 tokens_per_ep_rank=[49277, 48980, 49162, 49352, 49245, 49295, 48861, 49044] avg=49152.0000 max=49352 max_rank=3 min=48861 min_rank=6 imbalance_ratio=1.004069 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=13 tokens_per_ep_rank=[48999, 49247, 49160, 48954, 49311, 49408, 49168, 48969] avg=49152.0000 max=49408 max_rank=5 min=48954 min_rank=3 imbalance_ratio=1.005208 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=14 tokens_per_ep_rank=[49211, 49275, 49221, 49298, 48941, 49269, 49156, 48845] avg=49152.0000 max=49298 max_rank=3 min=48845 min_rank=7 imbalance_ratio=1.002970 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=15 tokens_per_ep_rank=[48889, 49107, 49011, 49120, 49329, 49125, 49066, 49569] avg=49152.0000 max=49569 max_rank=7 min=48889 min_rank=0 imbalance_ratio=1.008484 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=16 tokens_per_ep_rank=[49270, 49089, 49270, 49234, 49188, 48796, 48942, 49427] avg=49152.0000 max=49427 max_rank=7 min=48796 min_rank=5 imbalance_ratio=1.005595 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=17 tokens_per_ep_rank=[49160, 48895, 48998, 49456, 48967, 49010, 49242, 49488] avg=49152.0000 max=49488 max_rank=7 min=48895 min_rank=1 imbalance_ratio=1.006836 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=18 tokens_per_ep_rank=[49077, 49242, 48987, 49368, 49323, 49111, 49195, 48913] avg=49152.0000 max=49368 max_rank=3 min=48913 min_rank=7 imbalance_ratio=1.004395 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=19 tokens_per_ep_rank=[49549, 49270, 49258, 48703, 49069, 49033, 49367, 48967] avg=49152.0000 max=49549 max_rank=0 min=48703 min_rank=3 imbalance_ratio=1.008077 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=20 tokens_per_ep_rank=[49082, 49147, 48957, 49257, 49377, 49005, 48988, 49403] avg=49152.0000 max=49403 max_rank=7 min=48957 min_rank=2 imbalance_ratio=1.005107 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=21 tokens_per_ep_rank=[49069, 49091, 49069, 49433, 49017, 49462, 49000, 49075] avg=49152.0000 max=49462 max_rank=5 min=49000 min_rank=6 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=22 tokens_per_ep_rank=[48958, 48989, 49181, 49264, 48949, 49195, 49218, 49462] avg=49152.0000 max=49462 max_rank=7 min=48949 min_rank=4 imbalance_ratio=1.006307 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=23 total_expected_recv=1130088 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271104 peak_demand=49413 required_capacity=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=23 total_expected_recv=1128697 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273536 peak_demand=49389 required_capacity=1.004822 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=23 total_expected_recv=1129686 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272640 peak_demand=49418 required_capacity=1.005412 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=23 total_expected_recv=1131501 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269824 peak_demand=49554 required_capacity=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=23 total_expected_recv=1130917 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269824 peak_demand=49549 required_capacity=1.008077 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=23 tokens_per_ep_rank=[48928, 48776, 49238, 49401, 49325, 49490, 49041, 49017] avg=49152.0000 max=49490 max_rank=5 min=48776 min_rank=1 imbalance_ratio=1.006877 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=24 tokens_per_ep_rank=[49094, 49460, 49195, 48899, 48932, 49115, 49111, 49410] avg=49152.0000 max=49460 max_rank=1 min=48899 min_rank=3 imbalance_ratio=1.006266 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=23 total_expected_recv=1131660 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269440 peak_demand=49569 required_capacity=1.008484 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=23 total_expected_recv=1131104 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49456 required_capacity=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=23 total_expected_recv=1131501 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269824 peak_demand=49554 required_capacity=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=23 total_expected_recv=1128697 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273536 peak_demand=49389 required_capacity=1.004822 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=23 total_expected_recv=1130315 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49510 required_capacity=1.007284 - [2026-08-26 16:49:24] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 625.3 | throughput per GPU (TFLOP/s/GPU): 204.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.605448E+00 | loss scale: 1.0 | grad norm: 105.043 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48962 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49371 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49447 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=23 total_expected_recv=1131104 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49456 required_capacity=1.006185 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=23 total_expected_recv=1130917 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269824 peak_demand=49549 required_capacity=1.008077 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=23 total_expected_recv=1129686 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272640 peak_demand=49418 required_capacity=1.005412 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=23 total_expected_recv=1130088 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271104 peak_demand=49413 required_capacity=1.005310 + [2026-08-26 18:01:31] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 625.8 | throughput per GPU (TFLOP/s/GPU): 203.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.605448E+00 | loss scale: 1.0 | grad norm: 105.043 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48897 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49212 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49152 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48897 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48962 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49371 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49447 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49447 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49371 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48962 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48897 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49212 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48897 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49447 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49371 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49152 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=61440 expected_recv=48962 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49159 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49012 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49038 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49297 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=61440 expected_recv=48970 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49012 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49159 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49297 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49012 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49038 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49297 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49292 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49159 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=61440 expected_recv=48970 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49012 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49297 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49230 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49292 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49159 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=61440 expected_recv=48624 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49303 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49194 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=61440 expected_recv=48624 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49351 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49303 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49230 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49303 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49194 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=61440 expected_recv=48624 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49351 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49303 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=61440 expected_recv=48624 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49230 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49156 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49156 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49117 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48952 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49230 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48973 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49117 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49156 expected_overflow=0 exact_recv_per_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48952 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48973 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=61440 expected_recv=49156 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49124 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49117 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49117 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48952 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48926 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=61440 expected_recv=48952 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48895 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49124 expected_overflow=0 exact_recv_per_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48926 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48895 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=61440 expected_recv=49124 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49250 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49250 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48895 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49107 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49250 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48895 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48926 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49250 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49107 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48888 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49574 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49107 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49319 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48888 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49574 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49256 expected_overflow=0 exact_recv_per_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=61440 expected_recv=48926 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49319 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=61440 expected_recv=49256 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49221 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49213 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49362 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49574 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49221 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48806 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48888 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49574 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49213 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49362 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49213 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48806 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49197 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48888 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48806 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49165 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49482 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=61440 expected_recv=48806 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49165 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49482 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49213 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49197 expected_overflow=0 exact_recv_per_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49274 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=61440 expected_recv=49197 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49013 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49096 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49274 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49274 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49482 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49090 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49274 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49482 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49090 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49013 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49026 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49165 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49096 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49090 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49013 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49026 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49555 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49026 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49165 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49107 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49013 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49555 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49107 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49372 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49555 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48951 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49107 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49372 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49555 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48951 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49107 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49168 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48964 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48951 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48935 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=61440 expected_recv=49168 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49167 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=61440 expected_recv=48815 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48964 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48951 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48935 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49167 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=61440 expected_recv=48815 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48935 expected_overflow=0 exact_recv_per_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49354 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=61440 expected_recv=48815 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49583 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=61440 expected_recv=48935 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49167 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49354 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=61440 expected_recv=48815 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49583 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49167 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=61440 expected_recv=48830 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49583 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=61440 expected_recv=48896 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=61440 expected_recv=48830 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=61440 expected_recv=48830 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=61440 expected_recv=48896 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49129 expected_overflow=0 exact_recv_per_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49285 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49265 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=61440 expected_recv=48806 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=61440 expected_recv=48830 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=61440 expected_recv=49129 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=61440 expected_recv=48806 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49285 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=61440 expected_recv=48896 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49285 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49265 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49287 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=61440 expected_recv=48806 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=61440 expected_recv=48896 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49283 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49143 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49287 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49265 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49001 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49265 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49001 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49283 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49143 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=61440 expected_recv=48924 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49143 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49401 expected_overflow=0 exact_recv_per_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49182 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=10 calls=1 budget=61440 expected_recv=48924 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49182 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=61440 expected_recv=49401 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49280 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49143 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49193 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49493 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49283 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49193 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49280 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49283 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49182 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49182 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=10 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48826 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49493 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=61440 expected_recv=48990 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49184 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49526 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49184 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49526 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48826 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=61440 expected_recv=48990 expected_overflow=0 exact_recv_per_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=61440 expected_recv=48991 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48826 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49493 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=10 calls=1 budget=61440 expected_recv=48990 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48814 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49184 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49347 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48976 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=61440 expected_recv=48991 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49193 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49493 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48814 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49193 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49526 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48976 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49526 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49347 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=11 calls=1 budget=61440 expected_recv=48976 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49046 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49074 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49046 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49074 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49046 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49180 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49321 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49347 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49548 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=11 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=61440 expected_recv=48991 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49046 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49321 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49180 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49321 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49347 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=12 calls=1 budget=61440 expected_recv=48991 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49074 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49110 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49074 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49548 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49321 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49284 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49110 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=61440 expected_recv=48861 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49284 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=61440 expected_recv=48861 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=61440 expected_recv=48954 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49137 expected_overflow=0 exact_recv_per_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49284 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49039 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49548 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49327 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=13 calls=1 budget=61440 expected_recv=48954 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49180 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=12 calls=1 budget=61440 expected_recv=49137 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49284 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49039 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49548 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49327 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49180 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=61440 expected_recv=48861 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49039 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=61440 expected_recv=48982 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=13 calls=1 budget=61440 expected_recv=48861 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49039 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49230 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=13 calls=1 budget=61440 expected_recv=48982 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49230 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49385 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49368 expected_overflow=0 exact_recv_per_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49327 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49243 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49385 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=61440 expected_recv=48862 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=13 calls=1 budget=61440 expected_recv=49368 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49243 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49327 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=61440 expected_recv=48862 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49230 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49169 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49538 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=61440 expected_recv=48838 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49230 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49169 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49538 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=14 calls=1 budget=61440 expected_recv=48838 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49171 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49037 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49095 expected_overflow=0 exact_recv_per_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49538 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=61440 expected_recv=48862 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49171 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49243 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49037 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=14 calls=1 budget=61440 expected_recv=49095 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49538 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=15 calls=1 budget=61440 expected_recv=48862 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49222 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49177 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49243 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=61440 expected_recv=48962 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49222 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49177 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=61440 expected_recv=48700 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=15 calls=1 budget=61440 expected_recv=48962 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=61440 expected_recv=48700 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49212 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49304 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49212 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49089 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49041 expected_overflow=0 exact_recv_per_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49304 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49164 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49037 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49164 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49172 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49212 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49089 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=15 calls=1 budget=61440 expected_recv=49041 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=61440 expected_recv=48700 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49037 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49437 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49172 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49304 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=16 calls=1 budget=61440 expected_recv=48700 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49437 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49117 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49117 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49395 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49395 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49027 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49172 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49089 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49304 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49437 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49027 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49172 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49089 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49387 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49437 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49387 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49299 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49299 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48913 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48913 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49349 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49117 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48957 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49136 expected_overflow=0 exact_recv_per_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49349 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49117 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48957 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=16 calls=1 budget=61440 expected_recv=49136 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49395 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48901 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49349 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=61440 expected_recv=48930 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48957 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49395 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49387 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49256 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48901 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49349 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49330 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=61440 expected_recv=48930 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48957 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=17 calls=1 budget=61440 expected_recv=49387 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49256 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49330 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=61440 expected_recv=48978 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=61440 expected_recv=48978 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49518 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49518 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49212 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49303 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49299 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=61440 expected_recv=48924 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49131 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49212 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49303 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49131 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49299 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=61440 expected_recv=48930 expected_overflow=0 exact_recv_per_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=19 calls=1 budget=61440 expected_recv=48924 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48913 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=17 calls=1 budget=61440 expected_recv=48930 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49303 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49330 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49131 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=18 calls=1 budget=61440 expected_recv=48913 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49303 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49330 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49131 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=61440 expected_recv=48956 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=61440 expected_recv=48956 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49464 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49464 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=61440 expected_recv=48899 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=61440 expected_recv=48978 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49034 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49064 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49126 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=20 calls=1 budget=61440 expected_recv=48899 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49565 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=19 calls=1 budget=61440 expected_recv=48978 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49034 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49211 expected_overflow=0 exact_recv_per_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49080 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49518 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=18 calls=1 budget=61440 expected_recv=49211 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49024 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49064 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49080 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49518 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49064 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49126 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49024 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49565 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49052 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49257 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49036 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=61440 expected_recv=48956 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49079 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49175 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49257 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49036 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=20 calls=1 budget=61440 expected_recv=48956 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49079 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49126 expected_overflow=0 exact_recv_per_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49036 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49464 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=19 calls=1 budget=61440 expected_recv=49126 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49175 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49565 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49036 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49464 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49175 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49160 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49160 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49565 actual_valid_recv=50304 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49412 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49412 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49157 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49412 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49052 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49157 actual_valid_recv=49408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49066 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=20 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49204 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49486 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49157 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=61440 expected_recv=48897 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49066 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49486 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=61440 expected_recv=48897 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49204 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=21 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49160 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49167 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49167 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49160 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49486 expected_overflow=0 exact_recv_per_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=21 calls=1 budget=61440 expected_recv=49486 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=61440 expected_recv=48897 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=22 calls=1 budget=61440 expected_recv=48897 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49167 expected_overflow=0 exact_recv_per_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=22 calls=1 budget=61440 expected_recv=49167 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[49007, 49152, 48962, 49447, 48897, 49212, 49168, 49371] avg=49152.0000 max=49447 max_rank=3 min=48897 min_rank=4 imbalance_ratio=1.006002 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[49275, 49292, 49159, 49297, 49173, 49012, 48970, 49038] avg=49152.0000 max=49297 max_rank=3 min=48970 min_rank=6 imbalance_ratio=1.002950 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[49156, 49204, 49230, 49154, 48624, 49303, 49351, 49194] avg=49152.0000 max=49351 max_rank=6 min=48624 min_rank=4 imbalance_ratio=1.004049 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48950, 49117, 49646, 49221, 48973, 49128, 49057] avg=49152.0000 max=49646 max_rank=3 min=48950 min_rank=1 imbalance_ratio=1.010050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[49124, 48952, 49117, 49644, 49221, 48973, 49128, 49057] avg=49152.0000 max=49644 max_rank=3 min=48952 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000040 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[49256, 49107, 49250, 49319, 48926, 48895, 49263, 49200] avg=49152.0000 max=49319 max_rank=3 min=48895 min_rank=5 imbalance_ratio=1.003398 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[49197, 48806, 48955, 49213, 48888, 49574, 49362, 49221] avg=49152.0000 max=49574 max_rank=5 min=48806 min_rank=1 imbalance_ratio=1.008586 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[49070, 49026, 49096, 49013, 49165, 49482, 49090, 49274] avg=49152.0000 max=49482 max_rank=5 min=49013 min_rank=3 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[48935, 48964, 49107, 48951, 49168, 49555, 49164, 49372] avg=49152.0000 max=49555 max_rank=5 min=48935 min_rank=0 imbalance_ratio=1.008199 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[49129, 48830, 49167, 49080, 49258, 48815, 49583, 49354] avg=49152.0000 max=49583 max_rank=6 min=48815 min_rank=5 imbalance_ratio=1.008769 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=11 tokens_per_ep_rank=[49401, 49001, 48806, 49265, 49275, 48896, 49287, 49285] avg=49152.0000 max=49401 max_rank=0 min=48806 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=12 tokens_per_ep_rank=[48990, 49227, 49187, 49182, 49283, 49143, 49280, 48924] avg=49152.0000 max=49283 max_rank=4 min=48924 min_rank=7 imbalance_ratio=1.002665 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=13 tokens_per_ep_rank=[49204, 48976, 48826, 49526, 49193, 49493, 48814, 49184] avg=49152.0000 max=49526 max_rank=3 min=48814 min_rank=6 imbalance_ratio=1.007609 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=14 tokens_per_ep_rank=[49137, 49110, 49190, 49074, 48991, 49347, 49321, 49046] avg=49152.0000 max=49347 max_rank=5 min=48991 min_rank=4 imbalance_ratio=1.003967 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=15 tokens_per_ep_rank=[49368, 48982, 48954, 48861, 49180, 49548, 49039, 49284] avg=49152.0000 max=49548 max_rank=5 min=48861 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=16 tokens_per_ep_rank=[49095, 48838, 49385, 49230, 49327, 49128, 49169, 49044] avg=49152.0000 max=49385 max_rank=2 min=48838 min_rank=1 imbalance_ratio=1.004740 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=17 tokens_per_ep_rank=[49041, 48962, 49171, 49222, 49243, 48862, 49177, 49538] avg=49152.0000 max=49538 max_rank=7 min=48862 min_rank=5 imbalance_ratio=1.007853 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=18 tokens_per_ep_rank=[49136, 49437, 49212, 48700, 49037, 49226, 49304, 49164] avg=49152.0000 max=49437 max_rank=1 min=48700 min_rank=3 imbalance_ratio=1.005798 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=19 tokens_per_ep_rank=[48930, 49387, 49117, 49099, 49089, 49172, 49395, 49027] avg=49152.0000 max=49395 max_rank=6 min=48930 min_rank=0 imbalance_ratio=1.004944 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=20 tokens_per_ep_rank=[49211, 49330, 49299, 49256, 49349, 48957, 48913, 48901] avg=49152.0000 max=49349 max_rank=4 min=48901 min_rank=7 imbalance_ratio=1.004008 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=21 tokens_per_ep_rank=[49126, 49024, 48978, 49131, 49212, 48924, 49518, 49303] avg=49152.0000 max=49518 max_rank=6 min=48924 min_rank=5 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=22 tokens_per_ep_rank=[49154, 49565, 48956, 49064, 48899, 49034, 49464, 49080] avg=49152.0000 max=49565 max_rank=1 min=48899 min_rank=4 imbalance_ratio=1.008403 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=23 tokens_per_ep_rank=[49486, 48927, 49052, 49175, 49257, 49079, 49204, 49036] avg=49152.0000 max=49486 max_rank=0 min=48927 min_rank=1 imbalance_ratio=1.006795 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=24 tokens_per_ep_rank=[49167, 48897, 49160, 49099, 49412, 49157, 49258, 49066] avg=49152.0000 max=49412 max_rank=4 min=48897 min_rank=1 imbalance_ratio=1.005290 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=23 total_expected_recv=1132222 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268928 peak_demand=49583 required_capacity=1.008769 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=23 total_expected_recv=1129336 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272384 peak_demand=49385 required_capacity=1.004740 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=23 total_expected_recv=1130067 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270720 peak_demand=49412 required_capacity=1.005290 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=23 total_expected_recv=1130787 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270464 peak_demand=49574 required_capacity=1.008586 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=23 total_expected_recv=1130067 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270720 peak_demand=49412 required_capacity=1.005290 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=23 total_expected_recv=1130963 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270976 peak_demand=49538 required_capacity=1.007853 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=23 total_expected_recv=1129336 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272384 peak_demand=49385 required_capacity=1.004740 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=23 total_expected_recv=1131002 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270336 peak_demand=49644 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=23 total_expected_recv=1132222 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=268928 peak_demand=49583 required_capacity=1.008769 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=23 total_expected_recv=1128996 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=273152 peak_demand=49565 required_capacity=1.008403 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=23 total_expected_recv=1130595 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271360 peak_demand=49486 required_capacity=1.006795 - [2026-08-26 16:49:25] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 625.4 | throughput per GPU (TFLOP/s/GPU): 204.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.583566E+00 | loss scale: 1.0 | grad norm: 3.092 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49239 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49362 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] + [2026-08-26 18:01:31] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 627.9 | throughput per GPU (TFLOP/s/GPU): 203.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.583566E+00 | loss scale: 1.0 | grad norm: 3.092 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48753 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49059 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49364 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49258 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49239 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49362 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49362 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49364 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49059 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49111 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49239 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=61440 expected_recv=48753 actual_valid_recv=49024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49059 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49362 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49364 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49258 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49059 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49111 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49070 expected_overflow=0 exact_recv_per_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49239 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=61440 expected_recv=49070 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=61440 expected_recv=48966 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49199 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49410 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=61440 expected_recv=48966 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49199 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49410 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=61440 expected_recv=48966 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49010 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49199 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=61440 expected_recv=48966 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49010 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49322 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49270 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49190 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49410 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49322 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=61440 expected_recv=48849 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49010 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49270 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49190 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=61440 expected_recv=49410 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=61440 expected_recv=48849 expected_overflow=0 exact_recv_per_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49212 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48900 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49212 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=61440 expected_recv=48849 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48900 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49348 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48900 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49212 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49348 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49311 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49141 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48900 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49212 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48987 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49311 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49141 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49311 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=61440 expected_recv=48987 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49226 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=61440 expected_recv=48716 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49348 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49311 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49226 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49348 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=61440 expected_recv=48938 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49091 expected_overflow=0 exact_recv_per_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=61440 expected_recv=48716 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=61440 expected_recv=48938 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49511 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=61440 expected_recv=49091 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49023 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49511 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49287 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=61440 expected_recv=48716 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49023 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49023 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49511 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49434 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49263 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49287 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=61440 expected_recv=48716 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49023 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49434 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49263 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49035 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49503 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49035 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49503 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49047 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49035 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49434 expected_overflow=0 exact_recv_per_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49047 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49503 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49039 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=61440 expected_recv=49434 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49503 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=61440 expected_recv=48930 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49039 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49267 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49072 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=61440 expected_recv=48930 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49072 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49039 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49047 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=61440 expected_recv=48930 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49267 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49039 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49124 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49267 expected_overflow=0 exact_recv_per_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48955 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49124 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49029 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49230 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=61440 expected_recv=49267 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48955 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48609 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49029 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48609 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49230 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49529 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49586 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48609 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49230 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49029 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49529 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49586 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48870 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=61440 expected_recv=48609 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49029 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49187 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49243 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49187 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49243 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49243 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48870 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49586 expected_overflow=0 exact_recv_per_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49243 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=61440 expected_recv=49586 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48926 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48926 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49203 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48870 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48935 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49203 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49208 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48870 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48926 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49279 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48935 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49208 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=61440 expected_recv=48926 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49279 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49572 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49279 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49572 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49572 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49208 expected_overflow=0 exact_recv_per_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48847 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49009 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49572 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=61440 expected_recv=49208 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48847 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48894 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49323 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49009 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49308 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48894 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49323 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48984 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49009 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49459 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49308 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48847 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48984 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=61440 expected_recv=49009 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48847 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49173 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49301 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49173 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49459 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49301 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48984 expected_overflow=0 exact_recv_per_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=61440 expected_recv=48937 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49301 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=61440 expected_recv=48984 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=61440 expected_recv=48937 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49123 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49459 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=61440 expected_recv=48790 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49459 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49123 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49219 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=61440 expected_recv=48790 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49214 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49219 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=61440 expected_recv=48937 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=61440 expected_recv=48790 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49052 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49052 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49214 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=61440 expected_recv=48937 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49155 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49286 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49052 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49219 expected_overflow=0 exact_recv_per_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=61440 expected_recv=48952 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49155 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49286 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49051 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=61440 expected_recv=49219 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=61440 expected_recv=48952 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49101 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49051 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49336 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49283 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49101 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49283 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49336 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49175 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49283 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=61440 expected_recv=48952 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49155 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49116 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=61440 expected_recv=48952 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49155 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49116 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49175 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49175 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49114 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49336 expected_overflow=0 exact_recv_per_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49007 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49114 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=61440 expected_recv=48971 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49404 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49175 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49572 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=61440 expected_recv=49336 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49007 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=61440 expected_recv=48971 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=61440 expected_recv=48794 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49404 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=61440 expected_recv=48857 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49068 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=61440 expected_recv=48794 actual_valid_recv=49280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49572 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=10 calls=1 budget=61440 expected_recv=48857 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=61440 expected_recv=48794 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=61440 expected_recv=48971 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49114 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49256 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=11 calls=1 budget=61440 expected_recv=48794 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=10 calls=1 budget=61440 expected_recv=48971 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49114 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49256 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49068 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49073 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49068 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49073 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49478 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49478 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49456 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49014 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49572 expected_overflow=0 exact_recv_per_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49256 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49456 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49014 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49161 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49014 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49073 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=10 calls=1 budget=61440 expected_recv=49572 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49105 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49256 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49014 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49073 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=10 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49077 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=61440 expected_recv=48963 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49105 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49077 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=12 calls=1 budget=61440 expected_recv=48963 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49004 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49004 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49220 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48802 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49220 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49246 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49478 expected_overflow=0 exact_recv_per_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49161 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49363 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48966 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=11 calls=1 budget=61440 expected_recv=49478 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49154 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=11 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49363 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49161 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49154 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48802 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49220 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48802 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49246 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49438 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48966 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49363 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48937 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49220 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49438 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49142 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48937 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49142 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49357 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49494 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49357 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49253 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49593 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49494 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49004 expected_overflow=0 exact_recv_per_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49253 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=12 calls=1 budget=61440 expected_recv=49004 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49321 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=12 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48966 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48937 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49321 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49494 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48966 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=13 calls=1 budget=61440 expected_recv=48937 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49357 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49143 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49253 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49494 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49593 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49321 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49357 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49143 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48890 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48890 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48781 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49179 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48781 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48814 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49179 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=61440 expected_recv=48770 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49142 expected_overflow=0 exact_recv_per_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48814 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49574 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49366 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=13 calls=1 budget=61440 expected_recv=49142 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=61440 expected_recv=48770 actual_valid_recv=49152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=13 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49574 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48890 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49593 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49179 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49574 actual_valid_recv=50304 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49366 actual_valid_recv=50176 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48890 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=14 calls=1 budget=61440 expected_recv=49593 actual_valid_recv=49920 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=50304 slack_slots=11136 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49366 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49179 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48781 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49366 actual_valid_recv=50176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49188 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48781 actual_valid_recv=49152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48927 expected_overflow=0 exact_recv_per_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49009 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49188 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=14 calls=1 budget=61440 expected_recv=48927 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49330 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=14 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49099 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=61440 expected_recv=48770 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49279 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49009 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49188 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49330 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49099 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=15 calls=1 budget=61440 expected_recv=48770 actual_valid_recv=49152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49279 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49046 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49330 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49152 slack_slots=12288 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49188 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49333 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49046 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49330 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49009 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49165 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49333 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49092 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49275 expected_overflow=0 exact_recv_per_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49165 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=61440 expected_recv=48988 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49009 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49092 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=15 calls=1 budget=61440 expected_recv=49275 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49113 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=15 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49333 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49113 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49046 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49333 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=61440 expected_recv=48820 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49092 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49046 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=17 calls=1 budget=61440 expected_recv=48820 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49392 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49128 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49113 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49109 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49092 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49392 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49128 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49310 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49113 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49109 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=61440 expected_recv=48642 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=61440 expected_recv=48988 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49165 expected_overflow=0 exact_recv_per_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49310 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49454 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=61440 expected_recv=48642 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=16 calls=1 budget=61440 expected_recv=48988 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=16 calls=1 budget=61440 expected_recv=49165 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49361 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=16 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49361 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49109 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=61440 expected_recv=48999 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49454 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49392 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49109 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=18 calls=1 budget=61440 expected_recv=48999 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49143 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=61440 expected_recv=48642 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49392 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49384 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49208 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49361 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49143 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=17 calls=1 budget=61440 expected_recv=48642 actual_valid_recv=49024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49086 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49384 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49388 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49208 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49024 slack_slots=12416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49361 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49310 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49454 expected_overflow=0 exact_recv_per_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49086 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49388 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=61440 expected_recv=48808 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49310 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=17 calls=1 budget=61440 expected_recv=49454 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=17 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49208 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49224 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=61440 expected_recv=48808 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49143 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49208 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49224 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49086 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49256 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49251 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49143 actual_valid_recv=49792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48929 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49388 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=61440 expected_recv=48808 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48929 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49388 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49170 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=18 calls=1 budget=61440 expected_recv=48808 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49170 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48910 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49200 expected_overflow=0 exact_recv_per_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49269 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48910 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49256 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49269 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49290 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49256 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49086 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49251 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49290 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=61440 expected_recv=48659 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49076 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48910 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=18 calls=1 budget=61440 expected_recv=49200 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49076 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=18 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49400 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49269 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=61440 expected_recv=48659 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49029 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49076 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49400 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49269 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49029 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=61440 expected_recv=48940 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=61440 expected_recv=48906 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48929 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49290 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49458 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49170 expected_overflow=0 exact_recv_per_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49388 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=61440 expected_recv=48906 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=19 calls=1 budget=61440 expected_recv=48929 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49290 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=61440 expected_recv=48940 actual_valid_recv=49536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49458 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=19 calls=1 budget=61440 expected_recv=49170 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49388 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49332 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=61440 expected_recv=48906 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=19 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49332 actual_valid_recv=49920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49367 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=20 calls=1 budget=61440 expected_recv=48906 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49388 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49367 actual_valid_recv=50048 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49217 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=61440 expected_recv=48940 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49388 actual_valid_recv=50048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49217 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=21 calls=1 budget=61440 expected_recv=48940 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49034 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49332 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=61440 expected_recv=48659 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49123 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49192 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49029 expected_overflow=0 exact_recv_per_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49034 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49332 actual_valid_recv=49920 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=20 calls=1 budget=61440 expected_recv=48659 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49123 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49192 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=20 calls=1 budget=61440 expected_recv=49029 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49920 slack_slots=11520 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49107 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=20 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49034 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49107 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49034 actual_valid_recv=49536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49192 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49184 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49227 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49346 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49192 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49184 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49227 actual_valid_recv=49664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49346 actual_valid_recv=49792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49107 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=61440 expected_recv=48999 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49367 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49057 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=61440 expected_recv=48816 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49044 expected_overflow=0 exact_recv_per_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49107 actual_valid_recv=49664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=22 calls=1 budget=61440 expected_recv=48999 actual_valid_recv=49536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49367 actual_valid_recv=50048 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49057 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=61440 expected_recv=48816 actual_valid_recv=49280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=21 calls=1 budget=61440 expected_recv=49044 actual_valid_recv=49408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=50048 slack_slots=11392 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=61440 expected_recv=48816 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49020 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=21 calls=1 received_num_tokens=61440 valid_num_tokens=49408 slack_slots=12032 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49536 slack_slots=11904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=22 calls=1 budget=61440 expected_recv=48816 actual_valid_recv=49280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49020 actual_valid_recv=49664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49280 slack_slots=12160 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49664 slack_slots=11776 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49346 expected_overflow=0 exact_recv_per_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49644 actual_valid_recv=50176 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=22 calls=1 budget=61440 expected_recv=49346 actual_valid_recv=49792 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=50176 slack_slots=11264 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=22 calls=1 received_num_tokens=61440 valid_num_tokens=49792 slack_slots=11648 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[49070, 49111, 49239, 49059, 49258, 48753, 49362, 49364] avg=49152.0000 max=49364 max_rank=7 min=48753 min_rank=5 imbalance_ratio=1.004313 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[48849, 49190, 49410, 49322, 49270, 49199, 48966, 49010] avg=49152.0000 max=49410 max_rank=2 min=48849 min_rank=0 imbalance_ratio=1.005249 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[49091, 48987, 49141, 49311, 49226, 49212, 48900, 49348] avg=49152.0000 max=49348 max_rank=7 min=48900 min_rank=6 imbalance_ratio=1.003988 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[49434, 49287, 49511, 49023, 48716, 48938, 49044, 49263] avg=49152.0000 max=49511 max_rank=2 min=48716 min_rank=4 imbalance_ratio=1.007304 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[49267, 49323, 49072, 48930, 49047, 49035, 49503, 49039] avg=49152.0000 max=49503 max_rank=6 min=48930 min_rank=3 imbalance_ratio=1.007141 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[49586, 49529, 49230, 48609, 49154, 49124, 48955, 49029] avg=49152.0000 max=49586 max_rank=0 min=48609 min_rank=3 imbalance_ratio=1.008830 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48827, 49187, 49243, 49687] avg=49152.0000 max=49687 max_rank=7 min=48827 min_rank=4 imbalance_ratio=1.010885 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[49208, 48935, 49203, 48926, 48870, 49187, 49243, 49644] avg=49152.0000 max=49644 max_rank=7 min=48870 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=43 max_load_reduction_pct=0.000865 moved_to_replicas=43 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[48984, 49308, 48894, 49009, 49323, 49279, 49572, 48847] avg=49152.0000 max=49572 max_rank=6 min=48847 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[49219, 48790, 49123, 49214, 49459, 49173, 49301, 48937] avg=49152.0000 max=49459 max_rank=4 min=48790 min_rank=1 imbalance_ratio=1.006246 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=11 tokens_per_ep_rank=[49336, 49283, 49101, 48952, 49051, 49052, 49286, 49155] avg=49152.0000 max=49336 max_rank=0 min=48952 min_rank=3 imbalance_ratio=1.003743 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=12 tokens_per_ep_rank=[49572, 48857, 49404, 48971, 49175, 49116, 49007, 49114] avg=49152.0000 max=49572 max_rank=0 min=48857 min_rank=1 imbalance_ratio=1.008545 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=13 tokens_per_ep_rank=[49478, 49077, 49073, 49014, 49068, 48794, 49256, 49456] avg=49152.0000 max=49478 max_rank=0 min=48794 min_rank=5 imbalance_ratio=1.006632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=14 tokens_per_ep_rank=[49004, 49220, 49154, 49363, 48963, 49105, 49161, 49246] avg=49152.0000 max=49363 max_rank=3 min=48963 min_rank=4 imbalance_ratio=1.004293 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=15 tokens_per_ep_rank=[49142, 49357, 48937, 49321, 49438, 48802, 48966, 49253] avg=49152.0000 max=49438 max_rank=4 min=48802 min_rank=5 imbalance_ratio=1.005819 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=16 tokens_per_ep_rank=[48927, 48781, 48890, 49574, 49143, 49494, 49593, 48814] avg=49152.0000 max=49593 max_rank=6 min=48781 min_rank=1 imbalance_ratio=1.008972 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=17 tokens_per_ep_rank=[49275, 49009, 49099, 49330, 49366, 49179, 48770, 49188] avg=49152.0000 max=49366 max_rank=4 min=48770 min_rank=6 imbalance_ratio=1.004354 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=18 tokens_per_ep_rank=[49165, 48988, 49333, 49113, 49200, 49279, 49046, 49092] avg=49152.0000 max=49333 max_rank=2 min=48988 min_rank=1 imbalance_ratio=1.003682 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=19 tokens_per_ep_rank=[49310, 49454, 49109, 49361, 49128, 48820, 49392, 48642] avg=49152.0000 max=49454 max_rank=1 min=48642 min_rank=7 imbalance_ratio=1.006144 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=20 tokens_per_ep_rank=[49388, 48808, 49208, 49200, 49384, 48999, 49143, 49086] avg=49152.0000 max=49388 max_rank=0 min=48808 min_rank=1 imbalance_ratio=1.004801 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=21 tokens_per_ep_rank=[49076, 49400, 48910, 49170, 49251, 49224, 49256, 48929] avg=49152.0000 max=49400 max_rank=1 min=48910 min_rank=2 imbalance_ratio=1.005046 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=22 tokens_per_ep_rank=[49388, 49217, 49458, 49029, 49290, 49269, 48906, 48659] avg=49152.0000 max=49458 max_rank=2 min=48659 min_rank=7 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=23 tokens_per_ep_rank=[49192, 49184, 49123, 49044, 49332, 48940, 49034, 49367] avg=49152.0000 max=49367 max_rank=7 min=48940 min_rank=5 imbalance_ratio=1.004374 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[48789, 49020, 49057, 49346, 49107, 49227, 48999, 49671] avg=49152.0000 max=49671 max_rank=7 min=48789 min_rank=0 imbalance_ratio=1.010559 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=24 tokens_per_ep_rank=[48816, 49020, 49057, 49346, 49107, 49227, 48999, 49644] avg=49152.0000 max=49644 max_rank=7 min=48816 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=27 max_load_reduction_pct=0.000544 moved_to_replicas=27 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=23 total_expected_recv=1129200 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=272384 peak_demand=49494 required_capacity=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=23 total_expected_recv=1130126 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49644 required_capacity=1.010010 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=23 total_expected_recv=1131219 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270208 peak_demand=49459 required_capacity=1.006246 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=23 total_expected_recv=1131777 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269568 peak_demand=49586 required_capacity=1.008830 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=23 total_expected_recv=1130679 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270336 peak_demand=49511 required_capacity=1.007304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=23 total_expected_recv=1130191 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49574 required_capacity=1.008586 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=23 total_expected_recv=1130661 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49593 required_capacity=1.008972 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=23 total_expected_recv=1130115 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271488 peak_demand=49529 required_capacity=1.007670 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=23 total_expected_recv=1130126 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=270848 peak_demand=49644 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=23 total_expected_recv=1130191 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=271232 peak_demand=49574 required_capacity=1.008586 - [2026-08-26 16:49:26] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 624.1 | throughput per GPU (TFLOP/s/GPU): 204.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.183624E+00 | loss scale: 1.0 | grad norm: 1341.192 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | -[after training is done] datetime: 2026-08-26 16:49:26 -[rank0]:[W826 16:49:27.224616736 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W826 16:49:28.831636915 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W826 16:49:28.833255864 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W826 16:49:28.835597553 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W826 16:49:28.837688176 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W826 16:49:28.838801902 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W826 16:49:28.838976370 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W826 16:49:28.888714427 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0826 16:49:32.254000 270643 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 16:49:32.255000 270643 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 270733 closing signal SIGTERM -W0826 16:49:32.255000 270643 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 270734 closing signal SIGTERM -W0826 16:49:32.257000 270643 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 270735 closing signal SIGTERM -W0826 16:49:32.257000 270643 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 270736 closing signal SIGTERM -W0826 16:49:32.258000 270643 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 270737 closing signal SIGTERM -W0826 16:49:32.258000 270643 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 270738 closing signal SIGTERM -W0826 16:49:32.259000 270643 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 270739 closing signal SIGTERM -W0826 16:49:32.260000 270643 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 270740 closing signal SIGTERM +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=23 total_expected_recv=1131777 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=269568 peak_demand=49586 required_capacity=1.008830 + [2026-08-26 18:01:32] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 628.4 | throughput per GPU (TFLOP/s/GPU): 203.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.183624E+00 | loss scale: 1.0 | grad norm: 1341.192 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 50176 | min safe capacity: 1.0208 | +[after training is done] datetime: 2026-08-26 18:01:32 +[rank0]:[W826 18:01:33.636267597 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 18:01:34.939779678 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 18:01:34.940184948 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 18:01:34.941181014 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 18:01:34.941927362 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 18:01:34.041858159 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 18:01:35.117395911 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 18:01:35.143562794 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 18:01:39.114000 442964 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 18:01:39.115000 442964 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 443055 closing signal SIGTERM +W0826 18:01:39.115000 442964 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 443056 closing signal SIGTERM +W0826 18:01:39.117000 442964 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 443057 closing signal SIGTERM +W0826 18:01:39.117000 442964 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 443058 closing signal SIGTERM +W0826 18:01:39.117000 442964 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 443059 closing signal SIGTERM +W0826 18:01:39.119000 442964 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 443060 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -12746,4 +12744,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 270643 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 442964 got signal: 15 diff --git a/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/mem_samples.csv b/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..931e7d6ef7440bc093953de8311b407becb394c0 --- /dev/null +++ b/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/mem_samples.csv @@ -0,0 +1,200 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 725 +1, 725 +2, 725 +3, 725 +4, 725 +5, 725 +6, 725 +7, 725 +0, 1015 +1, 1029 +2, 1021 +3, 1033 +4, 1037 +5, 1051 +6, 1037 +7, 1005 +0, 1803 +1, 1803 +2, 1803 +3, 1803 +4, 1803 +5, 1803 +6, 1803 +7, 1803 +0, 2315 +1, 2319 +2, 2321 +3, 2327 +4, 2333 +5, 2329 +6, 2333 +7, 2323 +0, 46703 +1, 46137 +2, 46137 +3, 46837 +4, 46837 +5, 46837 +6, 46837 +7, 46837 +0, 47109 +1, 46537 +2, 46537 +3, 47237 +4, 47237 +5, 47237 +6, 47237 +7, 47237 +0, 48395 +1, 47825 +2, 47825 +3, 48525 +4, 48525 +5, 48525 +6, 48525 +7, 48525 +0, 48395 +1, 47825 +2, 47825 +3, 48525 +4, 48525 +5, 48525 +6, 48525 +7, 48525 +0, 48633 +1, 48063 +2, 48063 +3, 48763 +4, 48763 +5, 48763 +6, 48763 +7, 48763 +0, 50671 +1, 50101 +2, 50101 +3, 50801 +4, 50801 +5, 50801 +6, 50801 +7, 50801 +0, 73355 +1, 73327 +2, 73327 +3, 73339 +4, 73339 +5, 73339 +6, 73339 +7, 73339 +0, 74001 +1, 73973 +2, 73973 +3, 73985 +4, 73985 +5, 73985 +6, 73985 +7, 73985 +0, 74001 +1, 73973 +2, 73973 +3, 73985 +4, 73985 +5, 73985 +6, 73985 +7, 73985 +0, 74001 +1, 73975 +2, 73975 +3, 73987 +4, 73987 +5, 73987 +6, 73987 +7, 73987 +0, 74367 +1, 74363 +2, 74369 +3, 74389 +4, 74389 +5, 74383 +6, 74355 +7, 74383 +0, 82503 +1, 82149 +2, 82265 +3, 82265 +4, 82235 +5, 82253 +6, 82269 +7, 82267 +0, 96599 +1, 96629 +2, 96595 +3, 96633 +4, 96633 +5, 96633 +6, 96633 +7, 96633 +0, 96599 +1, 96629 +2, 96595 +3, 96633 +4, 96633 +5, 96633 +6, 96633 +7, 96633 +0, 96287 +1, 96119 +2, 96027 +3, 96385 +4, 96123 +5, 96065 +6, 96385 +7, 96385 +0, 3 +1, 1028 +2, 2564 +3, 2564 +4, 2564 +5, 2820 +6, 93171 +7, 3844 diff --git a/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/run.env b/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..83e889b7d94b5aeaba56e66fac1b58da6a5d911e --- /dev/null +++ b/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/run.env @@ -0,0 +1,61 @@ +backend=ultraep +config=QWEN2_57B +family=tierB +repo=/workspace/Megatron-UltraEP +branch=ultraep-token-drop-analysis +commit=eaf829e7 +num_layers=10 +hidden=3584 +ffn_hidden=2560 +moe_ffn_hidden=2560 +num_experts=64 +topk=8 +experts_per_rank=8 +expert_params=27525120 +heads=28 +kv_groups=4 +kv_channels= +qk_layernorm=0 +shared_ffn=20480 +shared_expert_overlap=0 +mla=0 +dense_first_layers=0 +moe_layers=10 +rotary_base=1000000 +rotary_scaling= +qkv_bias=1 +router_balancing=aux_loss +aux_loss_coeff=1e-2 +router_pre_softmax=1 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=8 +capacity_mode=static +capacity=1.0625 +cuda_graph=0 +probe_mode=1 +async_grad_reduce=0 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens=512 +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_recipe=mxfp8 diff --git a/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/train.log b/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..1096fbbb8e7c0aa2542bb28e5c378e5b8599bdc0 --- /dev/null +++ b/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/train.log @@ -0,0 +1,6093 @@ +[18:01:42] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +============================================================== + QWEN2_57B / ultraep (slots/rank=8 policy=matched) + E=64 K=8 H'=2560 m=8 + L=10 seq=4096 gbs=8 (1 microbatches) + cap_mode=static cap=1.0625 cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-UltraEP @ eaf829e7 +============================================================== +W0826 18:01:44.221000 446338 torch/distributed/run.py:766] +W0826 18:01:44.221000 446338 torch/distributed/run.py:766] ***************************************** +W0826 18:01:44.221000 446338 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 18:01:44.221000 446338 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 10 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 2560 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 3584 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_capacity_monitor ............................ True + moe_debug_drop_json ............................. None + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. True + moe_enable_ultraep .............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2560 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ None + moe_num_redundant_experts_per_rank .............. 8 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.0625 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 20480 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_ultraep_max_microbatches .................... None + moe_ultraep_overlap_weight_sync ................. True + moe_ultraep_weight_sync_join .................... before_dispatch + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 28 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 10 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 42 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 + +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 02436 is connected to is connected to is connected to is connected to is connected to 7777[Gloo] Rank 7 peer ranks. peer ranks. peer ranks. peer ranks. peer ranks. 7Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : is connected to 777 +777 + + + + peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0[Gloo] Rank is connected to 00 peer ranks. is connected to Expected number of connected peer ranks is : 00 peer ranks. +Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank2]:[W826 18:02:02.274828751 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 18:02:02.274828932 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 18:02:02.274879837 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 18:02:02.274926301 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 18:02:02.274996839 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 18:02:02.275010079 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 18:02:02.277009085 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 18:02:02.449467470 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 4.004 seconds +time to initialize megatron (seconds): 7.766 +[after megatron is initialized] datetime: 2026-08-26 18:02:08 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +[INFO] Use MNNVL fabric: False +[WARN] MAX_NUM_NVL_PEERS is not set. Using detected value 8. +NVSHMEM v3.7.2 +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +/dvs/p4/build/sw/rel/gpgpu/toolkit/r12.9/main_nvshmem/src/modules/transport/ibrc/ibrc.cpp:1754: error status: 7 (NVSHMEMX_ERROR_INTERNAL) Device enumeration failed. + +UltraEP: NVL domain size <= 8; forcing weight_sync plan mode to direct (was adaptive). + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 7262624256 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 22 +Params for bucket 1 (180289536 elements, 180289536 padded size): + module.output_layer.weight +Params for bucket 2 (73403904 elements, 73403904 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (146800640 elements, 146800640 padded size): + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight +Params for bucket 4 (103001600 elements, 103001600 padded size): + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.bias + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (146800640 elements, 146800640 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (103001600 elements, 103001600 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.bias + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.weight +Params for bucket 7 (146800640 elements, 146800640 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight +Params for bucket 8 (103001600 elements, 103001600 padded size): + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.bias + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight +Params for bucket 9 (146800640 elements, 146800640 padded size): + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 10 (103001600 elements, 103001600 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.self_attention.linear_qkv.bias + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight +Params for bucket 11 (146800640 elements, 146800640 padded size): + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (103001600 elements, 103001600 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.bias + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight +Params for bucket 13 (146800640 elements, 146800640 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight +Params for bucket 14 (103001600 elements, 103001600 padded size): + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.linear_qkv.bias + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight +Params for bucket 15 (146800640 elements, 146800640 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight +Params for bucket 16 (103001600 elements, 103001600 padded size): + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.bias + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 17 (146800640 elements, 146800640 padded size): + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (103001600 elements, 103001600 padded size): + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.bias + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight +Params for bucket 19 (146800640 elements, 146800640 padded size): + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight +Params for bucket 20 (103001600 elements, 103001600 padded size): + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.bias + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight +Params for bucket 21 (146800640 elements, 146800640 padded size): + module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight +Params for bucket 22 (209890816 elements, 209890816 padded size): + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_qkv.bias + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 +Params for bucket 1 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 +Params for bucket 2 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 +Params for bucket 3 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 4 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 5 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 +Params for bucket 6 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 +Params for bucket 7 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 +Params for bucket 8 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 +Params for bucket 9 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 10 (45875200 elements, 45875200 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 +Params for bucket 11 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 +Params for bucket 12 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 13 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 14 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 +Params for bucket 15 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 16 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 17 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 +Params for bucket 19 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 +Params for bucket 20 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 21 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 22 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 +Params for bucket 23 (45875200 elements, 45875200 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 24 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 +Params for bucket 25 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 26 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 +Params for bucket 27 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 +Params for bucket 28 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 29 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 +Params for bucket 30 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 31 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 +Params for bucket 32 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 33 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 34 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 35 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 +Params for bucket 36 (45875200 elements, 45875200 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 37 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 38 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 +Params for bucket 39 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 40 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 +Params for bucket 41 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 +Params for bucket 42 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 +Params for bucket 43 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +Params for bucket 44 (18350080 elements, 18350080 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:02:11 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 18:02:11 +done with setup ... +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3160.38, 3165.35) + train/valid/test-data-iterators-setup ..........: (66.68, 86.29) +[before the start of training step] datetime: 2026-08-26 18:02:11 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=34816 expected_recv=32916 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=34816 expected_recv=32984 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=34816 expected_recv=32984 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=34816 expected_recv=32916 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=34816 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=34816 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=34816 expected_recv=32879 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33098 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=34816 expected_recv=32912 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=34816 expected_recv=30584 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=34816 expected_recv=30584 actual_valid_recv=31104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=31104 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=34816 expected_recv=32238 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=34816 expected_recv=32238 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=34816 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=34816 expected_recv=32742 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=34816 expected_recv=32342 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=34816 expected_recv=32342 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=34816 expected_recv=31902 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=34816 expected_recv=31902 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=34816 expected_recv=31518 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32384 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=34816 expected_recv=31518 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=32000 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=34816 expected_recv=32116 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=34816 expected_recv=32116 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=34816 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=34816 expected_recv=32230 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=34816 expected_recv=32898 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=34816 expected_recv=32230 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=34816 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=34816 expected_recv=32731 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=34816 expected_recv=32421 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=34816 expected_recv=30836 expected_overflow=0 exact_recv_per_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=34816 expected_recv=32421 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=34816 expected_recv=32368 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=34816 expected_recv=32320 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=34816 expected_recv=32368 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=34816 expected_recv=32320 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=34816 expected_recv=32525 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=34816 expected_recv=30836 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=34816 expected_recv=32525 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=31616 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=34816 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=34816 expected_recv=32712 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=34816 expected_recv=32565 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=34816 expected_recv=32195 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=34816 expected_recv=32565 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=34816 expected_recv=32195 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=34816 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=34816 expected_recv=31390 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=34816 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=34816 expected_recv=31757 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=34816 expected_recv=32682 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=34816 expected_recv=31390 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=34816 expected_recv=31757 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=34816 expected_recv=32751 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=31872 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=34816 expected_recv=32249 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=34816 expected_recv=32249 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=34816 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=34816 expected_recv=32860 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=34816 expected_recv=32012 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=34816 expected_recv=32012 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=34816 expected_recv=31981 expected_overflow=0 exact_recv_per_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=34816 expected_recv=31981 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=34816 expected_recv=31123 expected_overflow=0 exact_recv_per_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=34816 expected_recv=31123 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=31616 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[28641, 44801, 16965, 27135, 32912, 21075, 64487, 26128] avg=32768.0000 max=64487 max_rank=6 min=16965 min_rank=2 imbalance_ratio=1.967987 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[33095, 33098, 33097, 33095, 32912, 30836, 33095, 32916] avg=32768.0000 max=33098 max_rank=1 min=30836 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=31389 max_load_reduction_pct=0.486749 moved_to_replicas=43096 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[29189, 51591, 24929, 33832, 27044, 18465, 47246, 29848] avg=32768.0000 max=51591 max_rank=1 min=18465 min_rank=5 imbalance_ratio=1.574432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32984, 33096, 33096, 33096, 33096, 33096, 33096, 30584] avg=32768.0000 max=33096 max_rank=1 min=30584 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=18495 max_load_reduction_pct=0.358493 moved_to_replicas=33381 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[32879, 31821, 25377, 31902, 47864, 28965, 37657, 25679] avg=32768.0000 max=47864 max_rank=4 min=25377 min_rank=2 imbalance_ratio=1.460693 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[32879, 32742, 33096, 31902, 33096, 33095, 33096, 32238] avg=32768.0000 max=33096 max_rank=2 min=31902 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=14768 max_load_reduction_pct=0.308541 moved_to_replicas=19329 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[37769, 34884, 24038, 35902, 32898, 26283, 33845, 36525] avg=32768.0000 max=37769 max_rank=0 min=24038 min_rank=2 imbalance_ratio=1.152618 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[33096, 33096, 31518, 33095, 32898, 32249, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31518 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=4673 max_load_reduction_pct=0.123726 moved_to_replicas=13445 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[38405, 21673, 48844, 35011, 32421, 29690, 22597, 33503] avg=32768.0000 max=48844 max_rank=2 min=21673 min_rank=1 imbalance_ratio=1.490601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 32421, 32012, 32230, 33096] avg=32768.0000 max=33097 max_rank=2 min=32012 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=15747 max_load_reduction_pct=0.322394 moved_to_replicas=23379 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=10 total_expected_recv=328324 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=15488 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[30997, 29107, 27145, 36308, 36997, 34441, 34418, 32731] avg=32768.0000 max=36997 max_rank=4 min=27145 min_rank=2 imbalance_ratio=1.129059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=10 total_expected_recv=326247 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16256 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[32342, 32320, 32368, 33095, 33096, 33096, 33096, 32731] avg=32768.0000 max=33096 max_rank=4 min=32320 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=3901 max_load_reduction_pct=0.105441 moved_to_replicas=9780 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[30244, 39523, 28897, 23534, 19933, 27954, 54033, 38026] avg=32768.0000 max=54033 max_rank=6 min=19933 min_rank=4 imbalance_ratio=1.648956 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[32116, 33096, 33097, 32565, 33096, 31981, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=31981 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=20936 max_load_reduction_pct=0.387467 moved_to_replicas=32294 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[37959, 41298, 41448, 31757, 28236, 25443, 31643, 24360] avg=32768.0000 max=41448 max_rank=2 min=24360 min_rank=7 imbalance_ratio=1.264893 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=10 total_expected_recv=329192 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=13312 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=10 total_expected_recv=326951 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=14976 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[33095, 33096, 33096, 31757, 33097, 33096, 32195, 32712] avg=32768.0000 max=33097 max_rank=4 min=31757 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=8351 max_load_reduction_pct=0.201481 moved_to_replicas=21417 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=10 total_expected_recv=329251 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=13184 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=10 total_expected_recv=329903 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=12928 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[28147, 23416, 31390, 57483, 23106, 25827, 37415, 35360] avg=32768.0000 max=57483 max_rank=3 min=23106 min_rank=4 imbalance_ratio=1.754242 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[32525, 32751, 31390, 33096, 33095, 33096, 33095, 33096] avg=32768.0000 max=33096 max_rank=3 min=31390 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24387 max_load_reduction_pct=0.424247 moved_to_replicas=30970 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=10 total_expected_recv=327892 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=14720 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[39453, 32860, 30450, 17734, 51691, 28327, 28947, 32682] avg=32768.0000 max=51691 max_rank=4 min=17734 min_rank=3 imbalance_ratio=1.577484 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[33096, 32860, 33096, 33095, 33096, 31123, 33096, 32682] avg=32768.0000 max=33096 max_rank=0 min=31123 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=18595 max_load_reduction_pct=0.359734 moved_to_replicas=24952 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=10 total_expected_recv=323680 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=18176 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 18:02:37] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 25995.4 | throughput per GPU (TFLOP/s/GPU): 4.8 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097922E+01 | loss scale: 1.0 | grad norm: 29.875 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +Number of parameters in transformer block in billions: 20.11 +Number of parameters in embedding layers in billions: 0.36 +Total number of parameters in billions: 20.47 +Number of parameters in most loaded shard in billions: 20.4724 +Theoretical memory footprints: weight and optimizer=146430.14 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 64260.6591796875 | max allocated: 65362.01904296875 | reserved: 73672.0 | max reserved: 73672.0 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=34816 expected_recv=32455 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=34816 expected_recv=32059 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33073 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=34816 expected_recv=32455 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=34816 expected_recv=32059 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33073 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=34816 expected_recv=32175 expected_overflow=0 exact_recv_per_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=34816 expected_recv=32175 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=34816 expected_recv=31975 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=34816 expected_recv=31975 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=34816 expected_recv=31592 expected_overflow=0 exact_recv_per_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=34816 expected_recv=31592 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=34816 expected_recv=31699 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=34816 expected_recv=31699 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=34816 expected_recv=31869 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=34816 expected_recv=31869 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32384 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=34816 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=34816 expected_recv=32569 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=34816 expected_recv=32393 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=34816 expected_recv=32393 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=34816 expected_recv=31702 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=34816 expected_recv=31702 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=34816 expected_recv=32547 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=34816 expected_recv=32547 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=34816 expected_recv=32194 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=34816 expected_recv=31869 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=34816 expected_recv=32194 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=34816 expected_recv=31923 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=34816 expected_recv=31869 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=34816 expected_recv=31923 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32384 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=34816 expected_recv=32084 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=34816 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=34816 expected_recv=32084 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=34816 expected_recv=32712 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=34816 expected_recv=31379 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=34816 expected_recv=31379 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=34816 expected_recv=31453 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=34816 expected_recv=32190 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=34816 expected_recv=31453 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=34816 expected_recv=32190 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=34816 expected_recv=32116 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=34816 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=34816 expected_recv=33099 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=34816 expected_recv=32116 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=34816 expected_recv=32813 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=34816 expected_recv=33099 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=34816 expected_recv=31396 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=34816 expected_recv=31396 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=31744 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=34816 expected_recv=32420 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=34816 expected_recv=32512 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=34816 expected_recv=32173 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=34816 expected_recv=32015 expected_overflow=0 exact_recv_per_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=34816 expected_recv=32420 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=34816 expected_recv=32512 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=34816 expected_recv=32173 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=34816 expected_recv=32015 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[21543, 43350, 18750, 27697, 33073, 22298, 68229, 27204] avg=32768.0000 max=68229 max_rank=6 min=18750 min_rank=2 imbalance_ratio=2.082184 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32175, 33096, 33095, 32059, 33073, 32455, 33095, 33096] avg=32768.0000 max=33096 max_rank=1 min=32059 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=35133 max_load_reduction_pct=0.514928 moved_to_replicas=45387 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[30213, 53507, 21851, 38313, 26641, 18445, 45417, 27757] avg=32768.0000 max=53507 max_rank=1 min=18445 min_rank=5 imbalance_ratio=1.632904 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[31592, 33096, 31975, 33096, 33096, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31592 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=20410 max_load_reduction_pct=0.381445 moved_to_replicas=37949 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[34036, 30016, 25522, 30561, 46110, 31699, 36060, 28140] avg=32768.0000 max=46110 max_rank=4 min=25522 min_rank=2 imbalance_ratio=1.407166 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 31869, 33096, 31699, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31699 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=13013 max_load_reduction_pct=0.282216 moved_to_replicas=16918 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[31702, 35630, 23563, 41135, 36051, 21996, 37717, 34350] avg=32768.0000 max=41135 max_rank=3 min=21996 min_rank=5 imbalance_ratio=1.255341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[31702, 33095, 32393, 33097, 33095, 32569, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31702 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=8038 max_load_reduction_pct=0.195405 moved_to_replicas=19403 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[38893, 22728, 45513, 34253, 31923, 32547, 19791, 36496] avg=32768.0000 max=45513 max_rank=2 min=19791 min_rank=6 imbalance_ratio=1.388947 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[33096, 33097, 33095, 33096, 31923, 32547, 32194, 33096] avg=32768.0000 max=33097 max_rank=1 min=31923 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=12416 max_load_reduction_pct=0.272801 moved_to_replicas=22771 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[27823, 32084, 28044, 28447, 36642, 44139, 35328, 29637] avg=32768.0000 max=44139 max_rank=5 min=27823 min_rank=0 imbalance_ratio=1.347015 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[33095, 32084, 33097, 32712, 33095, 33096, 33096, 31869] avg=32768.0000 max=33097 max_rank=2 min=31869 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=11042 max_load_reduction_pct=0.250164 moved_to_replicas=16821 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[26904, 42572, 31356, 19386, 18037, 24421, 61373, 38095] avg=32768.0000 max=61373 max_rank=6 min=18037 min_rank=4 imbalance_ratio=1.872955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[33096, 33096, 32190, 33096, 33096, 31379, 33095, 33096] avg=32768.0000 max=33096 max_rank=0 min=31379 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=28277 max_load_reduction_pct=0.460740 moved_to_replicas=42752 active_clone_links=9 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[33099, 42466, 47447, 28361, 26900, 25749, 30447, 27675] avg=32768.0000 max=47447 max_rank=2 min=25749 min_rank=5 imbalance_ratio=1.447968 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[33096, 33096, 33095, 33095, 33099, 33094, 32116, 31453] avg=32768.0000 max=33099 max_rank=4 min=31453 min_rank=7 imbalance_ratio=1.010101 max_load_reduction=14348 max_load_reduction_pct=0.302401 moved_to_replicas=23724 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[29557, 21481, 36051, 54601, 22125, 28431, 32420, 37478] avg=32768.0000 max=54601 max_rank=3 min=21481 min_rank=1 imbalance_ratio=1.666290 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[32512, 33096, 33096, 33096, 32015, 32813, 32420, 33096] avg=32768.0000 max=33096 max_rank=1 min=32015 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=21505 max_load_reduction_pct=0.393857 moved_to_replicas=28842 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[42711, 34406, 34421, 18717, 52867, 26323, 24989, 27710] avg=32768.0000 max=52867 max_rank=4 min=18717 min_rank=3 imbalance_ratio=1.613373 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=10 total_expected_recv=326390 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16256 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 33095, 32173, 33097, 31396] avg=32768.0000 max=33097 max_rank=6 min=31396 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19770 max_load_reduction_pct=0.373957 moved_to_replicas=32021 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=10 total_expected_recv=324921 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=17280 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=10 total_expected_recv=328311 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=13952 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=10 total_expected_recv=326556 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16256 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=10 total_expected_recv=329947 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=12544 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=10 total_expected_recv=328404 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=13440 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=10 total_expected_recv=328683 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=13312 peak_demand=33099 required_capacity=1.010101 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=10 total_expected_recv=328228 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=13952 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 18:02:38] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 363.3 | throughput per GPU (TFLOP/s/GPU): 339.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097874E+01 | loss scale: 1.0 | grad norm: 30.624 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33792 | min safe capacity: 1.0312 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=34816 expected_recv=31757 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33039 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=34816 expected_recv=31867 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=34816 expected_recv=31757 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33039 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=34816 expected_recv=31867 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=34816 expected_recv=31549 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=34816 expected_recv=32124 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=34816 expected_recv=32991 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=34816 expected_recv=31549 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=34816 expected_recv=32124 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=34816 expected_recv=32991 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=32000 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=34816 expected_recv=30844 expected_overflow=0 exact_recv_per_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=34816 expected_recv=30844 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=31488 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=34816 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=34816 expected_recv=32724 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=34816 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=34816 expected_recv=30472 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=31360 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=34816 expected_recv=33087 expected_overflow=0 exact_recv_per_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=34816 expected_recv=31316 expected_overflow=0 exact_recv_per_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=34816 expected_recv=33087 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=34816 expected_recv=32262 expected_overflow=0 exact_recv_per_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=34816 expected_recv=31316 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=34816 expected_recv=32262 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=31744 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=34816 expected_recv=32460 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=34816 expected_recv=32460 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=34816 expected_recv=31107 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=34816 expected_recv=31107 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=31872 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=34816 expected_recv=32543 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=34816 expected_recv=32543 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=34816 expected_recv=31387 expected_overflow=0 exact_recv_per_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=34816 expected_recv=31025 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=34816 expected_recv=31387 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=34816 expected_recv=31025 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=34816 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=34048 slack_slots=768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32000 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32000 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=34816 expected_recv=32535 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=34816 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=34816 expected_recv=30473 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=34816 expected_recv=32743 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=34816 expected_recv=30473 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=31360 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=34816 expected_recv=31974 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=34816 expected_recv=31974 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=34816 expected_recv=31593 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=34816 expected_recv=31593 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[19555, 34176, 14841, 35065, 30787, 33039, 67412, 27269] avg=32768.0000 max=67412 max_rank=6 min=14841 min_rank=2 imbalance_ratio=2.057251 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[19555, 34176, 14841, 35065, 30787, 33039, 67412, 27269] avg=32768.0000 max=67412 max_rank=6 min=14841 min_rank=2 imbalance_ratio=2.057251 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[19555, 34176, 14841, 35065, 30787, 33039, 67412, 27269] avg=32768.0000 max=67412 max_rank=6 min=14841 min_rank=2 imbalance_ratio=2.057251 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] avg=32768.0000 max=33097 max_rank=6 min=31757 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34315 max_load_reduction_pct=0.509034 moved_to_replicas=37365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[19555, 34176, 14841, 35065, 30787, 33039, 67412, 27269] avg=32768.0000 max=67412 max_rank=6 min=14841 min_rank=2 imbalance_ratio=2.057251 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] avg=32768.0000 max=33097 max_rank=6 min=31757 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34315 max_load_reduction_pct=0.509034 moved_to_replicas=37365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[19555, 34176, 14841, 35065, 30787, 33039, 67412, 27269] avg=32768.0000 max=67412 max_rank=6 min=14841 min_rank=2 imbalance_ratio=2.057251 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[19555, 34176, 14841, 35065, 30787, 33039, 67412, 27269] avg=32768.0000 max=67412 max_rank=6 min=14841 min_rank=2 imbalance_ratio=2.057251 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[19555, 34176, 14841, 35065, 30787, 33039, 67412, 27269] avg=32768.0000 max=67412 max_rank=6 min=14841 min_rank=2 imbalance_ratio=2.057251 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] avg=32768.0000 max=33097 max_rank=6 min=31757 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34315 max_load_reduction_pct=0.509034 moved_to_replicas=37365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] avg=32768.0000 max=33097 max_rank=6 min=31757 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34315 max_load_reduction_pct=0.509034 moved_to_replicas=37365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[19555, 34176, 14841, 35065, 30787, 33039, 67412, 27269] avg=32768.0000 max=67412 max_rank=6 min=14841 min_rank=2 imbalance_ratio=2.057251 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[17423, 68718, 15340, 27300, 32991, 14863, 41832, 43677] avg=32768.0000 max=68718 max_rank=1 min=14863 min_rank=5 imbalance_ratio=2.097107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] avg=32768.0000 max=33097 max_rank=6 min=31757 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34315 max_load_reduction_pct=0.509034 moved_to_replicas=37365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] avg=32768.0000 max=33097 max_rank=6 min=31757 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34315 max_load_reduction_pct=0.509034 moved_to_replicas=37365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] avg=32768.0000 max=33097 max_rank=6 min=31757 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34315 max_load_reduction_pct=0.509034 moved_to_replicas=37365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[17423, 68718, 15340, 27300, 32991, 14863, 41832, 43677] avg=32768.0000 max=68718 max_rank=1 min=14863 min_rank=5 imbalance_ratio=2.097107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31549 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=35621 max_load_reduction_pct=0.518365 moved_to_replicas=54939 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[17423, 68718, 15340, 27300, 32991, 14863, 41832, 43677] avg=32768.0000 max=68718 max_rank=1 min=14863 min_rank=5 imbalance_ratio=2.097107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 31867, 33039, 33097, 31757] avg=32768.0000 max=33097 max_rank=6 min=31757 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34315 max_load_reduction_pct=0.509034 moved_to_replicas=37365 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[17423, 68718, 15340, 27300, 32991, 14863, 41832, 43677] avg=32768.0000 max=68718 max_rank=1 min=14863 min_rank=5 imbalance_ratio=2.097107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[17423, 68718, 15340, 27300, 32991, 14863, 41832, 43677] avg=32768.0000 max=68718 max_rank=1 min=14863 min_rank=5 imbalance_ratio=2.097107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31549 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=35621 max_load_reduction_pct=0.518365 moved_to_replicas=54939 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[17423, 68718, 15340, 27300, 32991, 14863, 41832, 43677] avg=32768.0000 max=68718 max_rank=1 min=14863 min_rank=5 imbalance_ratio=2.097107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[17423, 68718, 15340, 27300, 32991, 14863, 41832, 43677] avg=32768.0000 max=68718 max_rank=1 min=14863 min_rank=5 imbalance_ratio=2.097107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31549 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=35621 max_load_reduction_pct=0.518365 moved_to_replicas=54939 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31549 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=35621 max_load_reduction_pct=0.518365 moved_to_replicas=54939 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[25273, 49553, 23828, 38338, 35093, 30844, 22175, 37040] avg=32768.0000 max=49553 max_rank=1 min=22175 min_rank=6 imbalance_ratio=1.512238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31549 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=35621 max_load_reduction_pct=0.518365 moved_to_replicas=54939 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[17423, 68718, 15340, 27300, 32991, 14863, 41832, 43677] avg=32768.0000 max=68718 max_rank=1 min=14863 min_rank=5 imbalance_ratio=2.097107 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31549 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=35621 max_load_reduction_pct=0.518365 moved_to_replicas=54939 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[25273, 49553, 23828, 38338, 35093, 30844, 22175, 37040] avg=32768.0000 max=49553 max_rank=1 min=22175 min_rank=6 imbalance_ratio=1.512238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31549 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=35621 max_load_reduction_pct=0.518365 moved_to_replicas=54939 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30844 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=16456 max_load_reduction_pct=0.332089 moved_to_replicas=27640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[33097, 33096, 32124, 33096, 32991, 31549, 33096, 33095] avg=32768.0000 max=33097 max_rank=0 min=31549 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=35621 max_load_reduction_pct=0.518365 moved_to_replicas=54939 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[25273, 49553, 23828, 38338, 35093, 30844, 22175, 37040] avg=32768.0000 max=49553 max_rank=1 min=22175 min_rank=6 imbalance_ratio=1.512238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[25273, 49553, 23828, 38338, 35093, 30844, 22175, 37040] avg=32768.0000 max=49553 max_rank=1 min=22175 min_rank=6 imbalance_ratio=1.512238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30844 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=16456 max_load_reduction_pct=0.332089 moved_to_replicas=27640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[25273, 49553, 23828, 38338, 35093, 30844, 22175, 37040] avg=32768.0000 max=49553 max_rank=1 min=22175 min_rank=6 imbalance_ratio=1.512238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[25273, 49553, 23828, 38338, 35093, 30844, 22175, 37040] avg=32768.0000 max=49553 max_rank=1 min=22175 min_rank=6 imbalance_ratio=1.512238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[25273, 49553, 23828, 38338, 35093, 30844, 22175, 37040] avg=32768.0000 max=49553 max_rank=1 min=22175 min_rank=6 imbalance_ratio=1.512238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[44611, 52747, 23240, 17381, 48227, 34918, 22359, 18661] avg=32768.0000 max=52747 max_rank=1 min=17381 min_rank=3 imbalance_ratio=1.609711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30844 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=16456 max_load_reduction_pct=0.332089 moved_to_replicas=27640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30844 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=16456 max_load_reduction_pct=0.332089 moved_to_replicas=27640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[25273, 49553, 23828, 38338, 35093, 30844, 22175, 37040] avg=32768.0000 max=49553 max_rank=1 min=22175 min_rank=6 imbalance_ratio=1.512238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30844 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=16456 max_load_reduction_pct=0.332089 moved_to_replicas=27640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30844 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=16456 max_load_reduction_pct=0.332089 moved_to_replicas=27640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19650 max_load_reduction_pct=0.372533 moved_to_replicas=48119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[44611, 52747, 23240, 17381, 48227, 34918, 22359, 18661] avg=32768.0000 max=52747 max_rank=1 min=17381 min_rank=3 imbalance_ratio=1.609711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30844 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=16456 max_load_reduction_pct=0.332089 moved_to_replicas=27640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[32724, 33095, 33096, 33096, 33096, 30844, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=30844 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=16456 max_load_reduction_pct=0.332089 moved_to_replicas=27640 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[44611, 52747, 23240, 17381, 48227, 34918, 22359, 18661] avg=32768.0000 max=52747 max_rank=1 min=17381 min_rank=3 imbalance_ratio=1.609711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[44611, 52747, 23240, 17381, 48227, 34918, 22359, 18661] avg=32768.0000 max=52747 max_rank=1 min=17381 min_rank=3 imbalance_ratio=1.609711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19650 max_load_reduction_pct=0.372533 moved_to_replicas=48119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[44611, 52747, 23240, 17381, 48227, 34918, 22359, 18661] avg=32768.0000 max=52747 max_rank=1 min=17381 min_rank=3 imbalance_ratio=1.609711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[23874, 8170, 67238, 9820, 37675, 36175, 31316, 47876] avg=32768.0000 max=67238 max_rank=2 min=8170 min_rank=1 imbalance_ratio=2.051941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[44611, 52747, 23240, 17381, 48227, 34918, 22359, 18661] avg=32768.0000 max=52747 max_rank=1 min=17381 min_rank=3 imbalance_ratio=1.609711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19650 max_load_reduction_pct=0.372533 moved_to_replicas=48119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19650 max_load_reduction_pct=0.372533 moved_to_replicas=48119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[44611, 52747, 23240, 17381, 48227, 34918, 22359, 18661] avg=32768.0000 max=52747 max_rank=1 min=17381 min_rank=3 imbalance_ratio=1.609711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[44611, 52747, 23240, 17381, 48227, 34918, 22359, 18661] avg=32768.0000 max=52747 max_rank=1 min=17381 min_rank=3 imbalance_ratio=1.609711 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19650 max_load_reduction_pct=0.372533 moved_to_replicas=48119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] avg=32768.0000 max=33096 max_rank=3 min=31316 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=34142 max_load_reduction_pct=0.507778 moved_to_replicas=56580 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[23874, 8170, 67238, 9820, 37675, 36175, 31316, 47876] avg=32768.0000 max=67238 max_rank=2 min=8170 min_rank=1 imbalance_ratio=2.051941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19650 max_load_reduction_pct=0.372533 moved_to_replicas=48119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19650 max_load_reduction_pct=0.372533 moved_to_replicas=48119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 33096, 33096, 33096, 30472] avg=32768.0000 max=33097 max_rank=2 min=30472 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=19650 max_load_reduction_pct=0.372533 moved_to_replicas=48119 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[23874, 8170, 67238, 9820, 37675, 36175, 31316, 47876] avg=32768.0000 max=67238 max_rank=2 min=8170 min_rank=1 imbalance_ratio=2.051941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[23874, 8170, 67238, 9820, 37675, 36175, 31316, 47876] avg=32768.0000 max=67238 max_rank=2 min=8170 min_rank=1 imbalance_ratio=2.051941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[23874, 8170, 67238, 9820, 37675, 36175, 31316, 47876] avg=32768.0000 max=67238 max_rank=2 min=8170 min_rank=1 imbalance_ratio=2.051941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] avg=32768.0000 max=33096 max_rank=3 min=31316 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=34142 max_load_reduction_pct=0.507778 moved_to_replicas=56580 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[47100, 53147, 32460, 24787, 36830, 18895, 35111, 13814] avg=32768.0000 max=53147 max_rank=1 min=13814 min_rank=7 imbalance_ratio=1.621918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] avg=32768.0000 max=33096 max_rank=3 min=31316 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=34142 max_load_reduction_pct=0.507778 moved_to_replicas=56580 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] avg=32768.0000 max=33096 max_rank=3 min=31316 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=34142 max_load_reduction_pct=0.507778 moved_to_replicas=56580 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[23874, 8170, 67238, 9820, 37675, 36175, 31316, 47876] avg=32768.0000 max=67238 max_rank=2 min=8170 min_rank=1 imbalance_ratio=2.051941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[23874, 8170, 67238, 9820, 37675, 36175, 31316, 47876] avg=32768.0000 max=67238 max_rank=2 min=8170 min_rank=1 imbalance_ratio=2.051941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] avg=32768.0000 max=33096 max_rank=3 min=31316 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=34142 max_load_reduction_pct=0.507778 moved_to_replicas=56580 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[23874, 8170, 67238, 9820, 37675, 36175, 31316, 47876] avg=32768.0000 max=67238 max_rank=2 min=8170 min_rank=1 imbalance_ratio=2.051941 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=0 min=31107 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20050 max_load_reduction_pct=0.377256 moved_to_replicas=39804 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[47100, 53147, 32460, 24787, 36830, 18895, 35111, 13814] avg=32768.0000 max=53147 max_rank=1 min=13814 min_rank=7 imbalance_ratio=1.621918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] avg=32768.0000 max=33096 max_rank=3 min=31316 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=34142 max_load_reduction_pct=0.507778 moved_to_replicas=56580 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] avg=32768.0000 max=33096 max_rank=3 min=31316 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=34142 max_load_reduction_pct=0.507778 moved_to_replicas=56580 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[32262, 33087, 33095, 33096, 33096, 33096, 31316, 33096] avg=32768.0000 max=33096 max_rank=3 min=31316 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=34142 max_load_reduction_pct=0.507778 moved_to_replicas=56580 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[47100, 53147, 32460, 24787, 36830, 18895, 35111, 13814] avg=32768.0000 max=53147 max_rank=1 min=13814 min_rank=7 imbalance_ratio=1.621918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=0 min=31107 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20050 max_load_reduction_pct=0.377256 moved_to_replicas=39804 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[47100, 53147, 32460, 24787, 36830, 18895, 35111, 13814] avg=32768.0000 max=53147 max_rank=1 min=13814 min_rank=7 imbalance_ratio=1.621918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[44886, 23234, 32543, 58109, 4518, 45984, 12014, 40856] avg=32768.0000 max=58109 max_rank=3 min=4518 min_rank=4 imbalance_ratio=1.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[47100, 53147, 32460, 24787, 36830, 18895, 35111, 13814] avg=32768.0000 max=53147 max_rank=1 min=13814 min_rank=7 imbalance_ratio=1.621918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=0 min=31107 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20050 max_load_reduction_pct=0.377256 moved_to_replicas=39804 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=0 min=31107 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20050 max_load_reduction_pct=0.377256 moved_to_replicas=39804 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[47100, 53147, 32460, 24787, 36830, 18895, 35111, 13814] avg=32768.0000 max=53147 max_rank=1 min=13814 min_rank=7 imbalance_ratio=1.621918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[47100, 53147, 32460, 24787, 36830, 18895, 35111, 13814] avg=32768.0000 max=53147 max_rank=1 min=13814 min_rank=7 imbalance_ratio=1.621918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[47100, 53147, 32460, 24787, 36830, 18895, 35111, 13814] avg=32768.0000 max=53147 max_rank=1 min=13814 min_rank=7 imbalance_ratio=1.621918 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[44886, 23234, 32543, 58109, 4518, 45984, 12014, 40856] avg=32768.0000 max=58109 max_rank=3 min=4518 min_rank=4 imbalance_ratio=1.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31025 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25012 max_load_reduction_pct=0.430432 moved_to_replicas=57451 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=0 min=31107 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20050 max_load_reduction_pct=0.377256 moved_to_replicas=39804 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=0 min=31107 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20050 max_load_reduction_pct=0.377256 moved_to_replicas=39804 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=0 min=31107 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20050 max_load_reduction_pct=0.377256 moved_to_replicas=39804 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[33097, 33096, 32460, 31107, 33096, 33095, 33097, 33096] avg=32768.0000 max=33097 max_rank=0 min=31107 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=20050 max_load_reduction_pct=0.377256 moved_to_replicas=39804 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[44886, 23234, 32543, 58109, 4518, 45984, 12014, 40856] avg=32768.0000 max=58109 max_rank=3 min=4518 min_rank=4 imbalance_ratio=1.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31025 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25012 max_load_reduction_pct=0.430432 moved_to_replicas=57451 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[44886, 23234, 32543, 58109, 4518, 45984, 12014, 40856] avg=32768.0000 max=58109 max_rank=3 min=4518 min_rank=4 imbalance_ratio=1.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[27843, 38320, 23437, 75416, 13156, 25082, 37996, 20894] avg=32768.0000 max=75416 max_rank=3 min=13156 min_rank=4 imbalance_ratio=2.301514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[44886, 23234, 32543, 58109, 4518, 45984, 12014, 40856] avg=32768.0000 max=58109 max_rank=3 min=4518 min_rank=4 imbalance_ratio=1.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31025 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25012 max_load_reduction_pct=0.430432 moved_to_replicas=57451 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[44886, 23234, 32543, 58109, 4518, 45984, 12014, 40856] avg=32768.0000 max=58109 max_rank=3 min=4518 min_rank=4 imbalance_ratio=1.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31025 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25012 max_load_reduction_pct=0.430432 moved_to_replicas=57451 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[44886, 23234, 32543, 58109, 4518, 45984, 12014, 40856] avg=32768.0000 max=58109 max_rank=3 min=4518 min_rank=4 imbalance_ratio=1.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] avg=32768.0000 max=33097 max_rank=3 min=31387 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42319 max_load_reduction_pct=0.561141 moved_to_replicas=52444 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31025 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25012 max_load_reduction_pct=0.430432 moved_to_replicas=57451 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[44886, 23234, 32543, 58109, 4518, 45984, 12014, 40856] avg=32768.0000 max=58109 max_rank=3 min=4518 min_rank=4 imbalance_ratio=1.773346 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[27843, 38320, 23437, 75416, 13156, 25082, 37996, 20894] avg=32768.0000 max=75416 max_rank=3 min=13156 min_rank=4 imbalance_ratio=2.301514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31025 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25012 max_load_reduction_pct=0.430432 moved_to_replicas=57451 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31025 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25012 max_load_reduction_pct=0.430432 moved_to_replicas=57451 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[33097, 33096, 32543, 33096, 31025, 33096, 33095, 33096] avg=32768.0000 max=33097 max_rank=0 min=31025 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=25012 max_load_reduction_pct=0.430432 moved_to_replicas=57451 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[27843, 38320, 23437, 75416, 13156, 25082, 37996, 20894] avg=32768.0000 max=75416 max_rank=3 min=13156 min_rank=4 imbalance_ratio=2.301514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[34924, 39797, 33444, 33461, 13309, 21582, 58232, 27395] avg=32768.0000 max=58232 max_rank=6 min=13309 min_rank=4 imbalance_ratio=1.777100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] avg=32768.0000 max=33097 max_rank=3 min=31387 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42319 max_load_reduction_pct=0.561141 moved_to_replicas=52444 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[27843, 38320, 23437, 75416, 13156, 25082, 37996, 20894] avg=32768.0000 max=75416 max_rank=3 min=13156 min_rank=4 imbalance_ratio=2.301514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[27843, 38320, 23437, 75416, 13156, 25082, 37996, 20894] avg=32768.0000 max=75416 max_rank=3 min=13156 min_rank=4 imbalance_ratio=2.301514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[27843, 38320, 23437, 75416, 13156, 25082, 37996, 20894] avg=32768.0000 max=75416 max_rank=3 min=13156 min_rank=4 imbalance_ratio=2.301514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] avg=32768.0000 max=33097 max_rank=4 min=30473 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25135 max_load_reduction_pct=0.431636 moved_to_replicas=34378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] avg=32768.0000 max=33097 max_rank=3 min=31387 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42319 max_load_reduction_pct=0.561141 moved_to_replicas=52444 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[27843, 38320, 23437, 75416, 13156, 25082, 37996, 20894] avg=32768.0000 max=75416 max_rank=3 min=13156 min_rank=4 imbalance_ratio=2.301514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] avg=32768.0000 max=33097 max_rank=3 min=31387 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42319 max_load_reduction_pct=0.561141 moved_to_replicas=52444 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] avg=32768.0000 max=33097 max_rank=3 min=31387 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42319 max_load_reduction_pct=0.561141 moved_to_replicas=52444 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[27843, 38320, 23437, 75416, 13156, 25082, 37996, 20894] avg=32768.0000 max=75416 max_rank=3 min=13156 min_rank=4 imbalance_ratio=2.301514 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[34924, 39797, 33444, 33461, 13309, 21582, 58232, 27395] avg=32768.0000 max=58232 max_rank=6 min=13309 min_rank=4 imbalance_ratio=1.777100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] avg=32768.0000 max=33097 max_rank=3 min=31387 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42319 max_load_reduction_pct=0.561141 moved_to_replicas=52444 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] avg=32768.0000 max=33097 max_rank=3 min=31387 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42319 max_load_reduction_pct=0.561141 moved_to_replicas=52444 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[14977, 13706, 3398, 45899, 41716, 52667, 17338, 72443] avg=32768.0000 max=72443 max_rank=7 min=3398 min_rank=2 imbalance_ratio=2.210785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[32743, 33095, 33095, 33097, 33096, 32535, 33096, 31387] avg=32768.0000 max=33097 max_rank=3 min=31387 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=42319 max_load_reduction_pct=0.561141 moved_to_replicas=52444 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] avg=32768.0000 max=33097 max_rank=4 min=30473 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25135 max_load_reduction_pct=0.431636 moved_to_replicas=34378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[34924, 39797, 33444, 33461, 13309, 21582, 58232, 27395] avg=32768.0000 max=58232 max_rank=6 min=13309 min_rank=4 imbalance_ratio=1.777100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[34924, 39797, 33444, 33461, 13309, 21582, 58232, 27395] avg=32768.0000 max=58232 max_rank=6 min=13309 min_rank=4 imbalance_ratio=1.777100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[34924, 39797, 33444, 33461, 13309, 21582, 58232, 27395] avg=32768.0000 max=58232 max_rank=6 min=13309 min_rank=4 imbalance_ratio=1.777100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] avg=32768.0000 max=33097 max_rank=1 min=31593 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=39346 max_load_reduction_pct=0.543130 moved_to_replicas=80341 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[34924, 39797, 33444, 33461, 13309, 21582, 58232, 27395] avg=32768.0000 max=58232 max_rank=6 min=13309 min_rank=4 imbalance_ratio=1.777100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] avg=32768.0000 max=33097 max_rank=4 min=30473 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25135 max_load_reduction_pct=0.431636 moved_to_replicas=34378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] avg=32768.0000 max=33097 max_rank=4 min=30473 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25135 max_load_reduction_pct=0.431636 moved_to_replicas=34378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[34924, 39797, 33444, 33461, 13309, 21582, 58232, 27395] avg=32768.0000 max=58232 max_rank=6 min=13309 min_rank=4 imbalance_ratio=1.777100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] avg=32768.0000 max=33097 max_rank=4 min=30473 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25135 max_load_reduction_pct=0.431636 moved_to_replicas=34378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[14977, 13706, 3398, 45899, 41716, 52667, 17338, 72443] avg=32768.0000 max=72443 max_rank=7 min=3398 min_rank=2 imbalance_ratio=2.210785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[34924, 39797, 33444, 33461, 13309, 21582, 58232, 27395] avg=32768.0000 max=58232 max_rank=6 min=13309 min_rank=4 imbalance_ratio=1.777100 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] avg=32768.0000 max=33097 max_rank=4 min=30473 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25135 max_load_reduction_pct=0.431636 moved_to_replicas=34378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] avg=32768.0000 max=33097 max_rank=4 min=30473 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25135 max_load_reduction_pct=0.431636 moved_to_replicas=34378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] avg=32768.0000 max=33097 max_rank=1 min=31593 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=39346 max_load_reduction_pct=0.543130 moved_to_replicas=80341 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[33095, 33095, 33096, 33096, 33097, 33096, 33096, 30473] avg=32768.0000 max=33097 max_rank=4 min=30473 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=25135 max_load_reduction_pct=0.431636 moved_to_replicas=34378 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[14977, 13706, 3398, 45899, 41716, 52667, 17338, 72443] avg=32768.0000 max=72443 max_rank=7 min=3398 min_rank=2 imbalance_ratio=2.210785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[14977, 13706, 3398, 45899, 41716, 52667, 17338, 72443] avg=32768.0000 max=72443 max_rank=7 min=3398 min_rank=2 imbalance_ratio=2.210785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[14977, 13706, 3398, 45899, 41716, 52667, 17338, 72443] avg=32768.0000 max=72443 max_rank=7 min=3398 min_rank=2 imbalance_ratio=2.210785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[14977, 13706, 3398, 45899, 41716, 52667, 17338, 72443] avg=32768.0000 max=72443 max_rank=7 min=3398 min_rank=2 imbalance_ratio=2.210785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] avg=32768.0000 max=33097 max_rank=1 min=31593 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=39346 max_load_reduction_pct=0.543130 moved_to_replicas=80341 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] avg=32768.0000 max=33097 max_rank=1 min=31593 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=39346 max_load_reduction_pct=0.543130 moved_to_replicas=80341 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[14977, 13706, 3398, 45899, 41716, 52667, 17338, 72443] avg=32768.0000 max=72443 max_rank=7 min=3398 min_rank=2 imbalance_ratio=2.210785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] avg=32768.0000 max=33097 max_rank=1 min=31593 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=39346 max_load_reduction_pct=0.543130 moved_to_replicas=80341 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[14977, 13706, 3398, 45899, 41716, 52667, 17338, 72443] avg=32768.0000 max=72443 max_rank=7 min=3398 min_rank=2 imbalance_ratio=2.210785 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] avg=32768.0000 max=33097 max_rank=1 min=31593 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=39346 max_load_reduction_pct=0.543130 moved_to_replicas=80341 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] avg=32768.0000 max=33097 max_rank=1 min=31593 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=39346 max_load_reduction_pct=0.543130 moved_to_replicas=80341 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[33095, 33097, 31974, 33096, 33096, 33097, 31593, 33096] avg=32768.0000 max=33097 max_rank=1 min=31593 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=39346 max_load_reduction_pct=0.543130 moved_to_replicas=80341 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=10 total_expected_recv=322664 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=18944 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=10 total_expected_recv=326543 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=15744 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=10 total_expected_recv=329401 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=12544 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=10 total_expected_recv=327676 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=14336 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=10 total_expected_recv=330949 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=10752 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=10 total_expected_recv=327679 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=14592 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=10 total_expected_recv=328972 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=12928 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=10 total_expected_recv=327556 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=14080 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 18:02:38] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 243.2 | throughput per GPU (TFLOP/s/GPU): 507.7 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.539555E+00 | loss scale: 1.0 | grad norm: 32.434 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33792 | min safe capacity: 1.0312 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=34816 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=34816 expected_recv=30920 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=34816 expected_recv=32648 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=34816 expected_recv=30920 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=31360 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=34816 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=34816 expected_recv=30628 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=34816 expected_recv=32940 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=34816 expected_recv=30628 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=34048 slack_slots=768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=31488 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=34816 expected_recv=31583 expected_overflow=0 exact_recv_per_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=34816 expected_recv=31583 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=34816 expected_recv=32000 expected_overflow=0 exact_recv_per_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=34816 expected_recv=33080 expected_overflow=0 exact_recv_per_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32384 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=34816 expected_recv=32000 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=34816 expected_recv=33080 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=34816 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=34816 expected_recv=30853 expected_overflow=0 exact_recv_per_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=34816 expected_recv=32715 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=34816 expected_recv=30853 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=31744 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=34816 expected_recv=30861 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=34816 expected_recv=30861 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=34816 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=31616 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=34816 expected_recv=32706 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=34816 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=34176 slack_slots=640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=34816 expected_recv=32764 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=34816 expected_recv=32139 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=34816 expected_recv=32139 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=34816 expected_recv=32104 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34176 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=34816 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=34816 expected_recv=32104 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=34176 slack_slots=640 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=34816 expected_recv=32557 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=34816 expected_recv=32753 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=34816 expected_recv=31683 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=34816 expected_recv=32557 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=34816 expected_recv=31683 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=34816 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=34816 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=34816 expected_recv=31736 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=34816 expected_recv=32733 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=34816 expected_recv=32787 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=34816 expected_recv=31736 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=34048 slack_slots=768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=34816 expected_recv=32167 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=34816 expected_recv=32167 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=34816 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=34816 expected_recv=32760 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=34816 expected_recv=32070 expected_overflow=0 exact_recv_per_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=34816 expected_recv=30912 expected_overflow=0 exact_recv_per_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=34816 expected_recv=32070 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=34816 expected_recv=31498 expected_overflow=0 exact_recv_per_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=34816 expected_recv=30912 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=34048 slack_slots=768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=31488 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=34816 expected_recv=31498 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[24002, 20052, 20606, 43131, 23808, 44347, 57985, 28213] avg=32768.0000 max=57985 max_rank=6 min=20052 min_rank=1 imbalance_ratio=1.769562 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[24002, 20052, 20606, 43131, 23808, 44347, 57985, 28213] avg=32768.0000 max=57985 max_rank=6 min=20052 min_rank=1 imbalance_ratio=1.769562 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] avg=32768.0000 max=33096 max_rank=0 min=30920 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24889 max_load_reduction_pct=0.429232 moved_to_replicas=46175 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=34048 slack_slots=768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=34816 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=34816 expected_recv=32656 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[24002, 20052, 20606, 43131, 23808, 44347, 57985, 28213] avg=32768.0000 max=57985 max_rank=6 min=20052 min_rank=1 imbalance_ratio=1.769562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] avg=32768.0000 max=33096 max_rank=0 min=30920 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24889 max_load_reduction_pct=0.429232 moved_to_replicas=46175 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[24002, 20052, 20606, 43131, 23808, 44347, 57985, 28213] avg=32768.0000 max=57985 max_rank=6 min=20052 min_rank=1 imbalance_ratio=1.769562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[24002, 20052, 20606, 43131, 23808, 44347, 57985, 28213] avg=32768.0000 max=57985 max_rank=6 min=20052 min_rank=1 imbalance_ratio=1.769562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[63715, 25026, 19622, 32940, 19636, 30317, 25717, 45171] avg=32768.0000 max=63715 max_rank=0 min=19622 min_rank=2 imbalance_ratio=1.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[24002, 20052, 20606, 43131, 23808, 44347, 57985, 28213] avg=32768.0000 max=57985 max_rank=6 min=20052 min_rank=1 imbalance_ratio=1.769562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] avg=32768.0000 max=33096 max_rank=0 min=30920 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24889 max_load_reduction_pct=0.429232 moved_to_replicas=46175 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] avg=32768.0000 max=33096 max_rank=0 min=30920 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24889 max_load_reduction_pct=0.429232 moved_to_replicas=46175 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[24002, 20052, 20606, 43131, 23808, 44347, 57985, 28213] avg=32768.0000 max=57985 max_rank=6 min=20052 min_rank=1 imbalance_ratio=1.769562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[63715, 25026, 19622, 32940, 19636, 30317, 25717, 45171] avg=32768.0000 max=63715 max_rank=0 min=19622 min_rank=2 imbalance_ratio=1.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] avg=32768.0000 max=33096 max_rank=0 min=30920 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24889 max_load_reduction_pct=0.429232 moved_to_replicas=46175 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30628 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30619 max_load_reduction_pct=0.480562 moved_to_replicas=42694 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[24002, 20052, 20606, 43131, 23808, 44347, 57985, 28213] avg=32768.0000 max=57985 max_rank=6 min=20052 min_rank=1 imbalance_ratio=1.769562 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] avg=32768.0000 max=33096 max_rank=0 min=30920 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24889 max_load_reduction_pct=0.429232 moved_to_replicas=46175 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[63715, 25026, 19622, 32940, 19636, 30317, 25717, 45171] avg=32768.0000 max=63715 max_rank=0 min=19622 min_rank=2 imbalance_ratio=1.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30628 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30619 max_load_reduction_pct=0.480562 moved_to_replicas=42694 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] avg=32768.0000 max=33096 max_rank=0 min=30920 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24889 max_load_reduction_pct=0.429232 moved_to_replicas=46175 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[33096, 33096, 30920, 33096, 33096, 33096, 33096, 32648] avg=32768.0000 max=33096 max_rank=0 min=30920 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=24889 max_load_reduction_pct=0.429232 moved_to_replicas=46175 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[43706, 33080, 31395, 13178, 58219, 33701, 40301, 8564] avg=32768.0000 max=58219 max_rank=4 min=8564 min_rank=7 imbalance_ratio=1.776703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[63715, 25026, 19622, 32940, 19636, 30317, 25717, 45171] avg=32768.0000 max=63715 max_rank=0 min=19622 min_rank=2 imbalance_ratio=1.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[63715, 25026, 19622, 32940, 19636, 30317, 25717, 45171] avg=32768.0000 max=63715 max_rank=0 min=19622 min_rank=2 imbalance_ratio=1.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[63715, 25026, 19622, 32940, 19636, 30317, 25717, 45171] avg=32768.0000 max=63715 max_rank=0 min=19622 min_rank=2 imbalance_ratio=1.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30628 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30619 max_load_reduction_pct=0.480562 moved_to_replicas=42694 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31583 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25122 max_load_reduction_pct=0.431509 moved_to_replicas=43543 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[43706, 33080, 31395, 13178, 58219, 33701, 40301, 8564] avg=32768.0000 max=58219 max_rank=4 min=8564 min_rank=7 imbalance_ratio=1.776703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[63715, 25026, 19622, 32940, 19636, 30317, 25717, 45171] avg=32768.0000 max=63715 max_rank=0 min=19622 min_rank=2 imbalance_ratio=1.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[63715, 25026, 19622, 32940, 19636, 30317, 25717, 45171] avg=32768.0000 max=63715 max_rank=0 min=19622 min_rank=2 imbalance_ratio=1.944427 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30628 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30619 max_load_reduction_pct=0.480562 moved_to_replicas=42694 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30628 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30619 max_load_reduction_pct=0.480562 moved_to_replicas=42694 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30628 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30619 max_load_reduction_pct=0.480562 moved_to_replicas=42694 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[43706, 33080, 31395, 13178, 58219, 33701, 40301, 8564] avg=32768.0000 max=58219 max_rank=4 min=8564 min_rank=7 imbalance_ratio=1.776703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31583 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25122 max_load_reduction_pct=0.431509 moved_to_replicas=43543 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30628 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30619 max_load_reduction_pct=0.480562 moved_to_replicas=42694 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32940, 33096, 30628, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30628 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=30619 max_load_reduction_pct=0.480562 moved_to_replicas=42694 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[28621, 32715, 2104, 11548, 41606, 19390, 39631, 86529] avg=32768.0000 max=86529 max_rank=7 min=2104 min_rank=2 imbalance_ratio=2.640656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[43706, 33080, 31395, 13178, 58219, 33701, 40301, 8564] avg=32768.0000 max=58219 max_rank=4 min=8564 min_rank=7 imbalance_ratio=1.776703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[43706, 33080, 31395, 13178, 58219, 33701, 40301, 8564] avg=32768.0000 max=58219 max_rank=4 min=8564 min_rank=7 imbalance_ratio=1.776703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[43706, 33080, 31395, 13178, 58219, 33701, 40301, 8564] avg=32768.0000 max=58219 max_rank=4 min=8564 min_rank=7 imbalance_ratio=1.776703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31583 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25122 max_load_reduction_pct=0.431509 moved_to_replicas=43543 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30853 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53433 max_load_reduction_pct=0.617516 moved_to_replicas=68478 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[28621, 32715, 2104, 11548, 41606, 19390, 39631, 86529] avg=32768.0000 max=86529 max_rank=7 min=2104 min_rank=2 imbalance_ratio=2.640656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[43706, 33080, 31395, 13178, 58219, 33701, 40301, 8564] avg=32768.0000 max=58219 max_rank=4 min=8564 min_rank=7 imbalance_ratio=1.776703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[43706, 33080, 31395, 13178, 58219, 33701, 40301, 8564] avg=32768.0000 max=58219 max_rank=4 min=8564 min_rank=7 imbalance_ratio=1.776703 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31583 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25122 max_load_reduction_pct=0.431509 moved_to_replicas=43543 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31583 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25122 max_load_reduction_pct=0.431509 moved_to_replicas=43543 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31583 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25122 max_load_reduction_pct=0.431509 moved_to_replicas=43543 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30853 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53433 max_load_reduction_pct=0.617516 moved_to_replicas=68478 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[28621, 32715, 2104, 11548, 41606, 19390, 39631, 86529] avg=32768.0000 max=86529 max_rank=7 min=2104 min_rank=2 imbalance_ratio=2.640656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[45811, 9502, 98383, 22972, 38922, 13297, 51, 33206] avg=32768.0000 max=98383 max_rank=2 min=51 min_rank=6 imbalance_ratio=3.002411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31583 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25122 max_load_reduction_pct=0.431509 moved_to_replicas=43543 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[33097, 33080, 32000, 31583, 33096, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=31583 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=25122 max_load_reduction_pct=0.431509 moved_to_replicas=43543 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[28621, 32715, 2104, 11548, 41606, 19390, 39631, 86529] avg=32768.0000 max=86529 max_rank=7 min=2104 min_rank=2 imbalance_ratio=2.640656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30853 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53433 max_load_reduction_pct=0.617516 moved_to_replicas=68478 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[28621, 32715, 2104, 11548, 41606, 19390, 39631, 86529] avg=32768.0000 max=86529 max_rank=7 min=2104 min_rank=2 imbalance_ratio=2.640656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] avg=32768.0000 max=33097 max_rank=0 min=30861 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65286 max_load_reduction_pct=0.663590 moved_to_replicas=83938 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[28621, 32715, 2104, 11548, 41606, 19390, 39631, 86529] avg=32768.0000 max=86529 max_rank=7 min=2104 min_rank=2 imbalance_ratio=2.640656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[45811, 9502, 98383, 22972, 38922, 13297, 51, 33206] avg=32768.0000 max=98383 max_rank=2 min=51 min_rank=6 imbalance_ratio=3.002411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[28621, 32715, 2104, 11548, 41606, 19390, 39631, 86529] avg=32768.0000 max=86529 max_rank=7 min=2104 min_rank=2 imbalance_ratio=2.640656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[28621, 32715, 2104, 11548, 41606, 19390, 39631, 86529] avg=32768.0000 max=86529 max_rank=7 min=2104 min_rank=2 imbalance_ratio=2.640656 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30853 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53433 max_load_reduction_pct=0.617516 moved_to_replicas=68478 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30853 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53433 max_load_reduction_pct=0.617516 moved_to_replicas=68478 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30853 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53433 max_load_reduction_pct=0.617516 moved_to_replicas=68478 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[45811, 9502, 98383, 22972, 38922, 13297, 51, 33206] avg=32768.0000 max=98383 max_rank=2 min=51 min_rank=6 imbalance_ratio=3.002411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32104, 65748, 29780, 22843, 68039, 32753, 10607, 270] avg=32768.0000 max=68039 max_rank=4 min=270 min_rank=7 imbalance_ratio=2.076385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] avg=32768.0000 max=33097 max_rank=0 min=30861 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65286 max_load_reduction_pct=0.663590 moved_to_replicas=83938 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30853 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53433 max_load_reduction_pct=0.617516 moved_to_replicas=68478 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[33096, 32715, 33096, 33096, 33096, 30853, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=30853 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=53433 max_load_reduction_pct=0.617516 moved_to_replicas=68478 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[45811, 9502, 98383, 22972, 38922, 13297, 51, 33206] avg=32768.0000 max=98383 max_rank=2 min=51 min_rank=6 imbalance_ratio=3.002411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[45811, 9502, 98383, 22972, 38922, 13297, 51, 33206] avg=32768.0000 max=98383 max_rank=2 min=51 min_rank=6 imbalance_ratio=3.002411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] avg=32768.0000 max=33097 max_rank=0 min=30861 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65286 max_load_reduction_pct=0.663590 moved_to_replicas=83938 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34943 max_load_reduction_pct=0.513573 moved_to_replicas=67595 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[45811, 9502, 98383, 22972, 38922, 13297, 51, 33206] avg=32768.0000 max=98383 max_rank=2 min=51 min_rank=6 imbalance_ratio=3.002411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32104, 65748, 29780, 22843, 68039, 32753, 10607, 270] avg=32768.0000 max=68039 max_rank=4 min=270 min_rank=7 imbalance_ratio=2.076385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[45811, 9502, 98383, 22972, 38922, 13297, 51, 33206] avg=32768.0000 max=98383 max_rank=2 min=51 min_rank=6 imbalance_ratio=3.002411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[45811, 9502, 98383, 22972, 38922, 13297, 51, 33206] avg=32768.0000 max=98383 max_rank=2 min=51 min_rank=6 imbalance_ratio=3.002411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] avg=32768.0000 max=33097 max_rank=0 min=30861 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65286 max_load_reduction_pct=0.663590 moved_to_replicas=83938 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] avg=32768.0000 max=33097 max_rank=0 min=30861 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65286 max_load_reduction_pct=0.663590 moved_to_replicas=83938 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] avg=32768.0000 max=33097 max_rank=0 min=30861 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65286 max_load_reduction_pct=0.663590 moved_to_replicas=83938 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[72691, 50187, 32733, 31683, 43, 69, 74714, 24] avg=32768.0000 max=74714 max_rank=6 min=24 min_rank=7 imbalance_ratio=2.280090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34943 max_load_reduction_pct=0.513573 moved_to_replicas=67595 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] avg=32768.0000 max=33097 max_rank=0 min=30861 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65286 max_load_reduction_pct=0.663590 moved_to_replicas=83938 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[33097, 33096, 33096, 30861, 33097, 33095, 32706, 33096] avg=32768.0000 max=33097 max_rank=0 min=30861 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65286 max_load_reduction_pct=0.663590 moved_to_replicas=83938 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32104, 65748, 29780, 22843, 68039, 32753, 10607, 270] avg=32768.0000 max=68039 max_rank=4 min=270 min_rank=7 imbalance_ratio=2.076385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32104, 65748, 29780, 22843, 68039, 32753, 10607, 270] avg=32768.0000 max=68039 max_rank=4 min=270 min_rank=7 imbalance_ratio=2.076385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32104, 65748, 29780, 22843, 68039, 32753, 10607, 270] avg=32768.0000 max=68039 max_rank=4 min=270 min_rank=7 imbalance_ratio=2.076385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] avg=32768.0000 max=33097 max_rank=0 min=31683 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41617 max_load_reduction_pct=0.557017 moved_to_replicas=98304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32104, 65748, 29780, 22843, 68039, 32753, 10607, 270] avg=32768.0000 max=68039 max_rank=4 min=270 min_rank=7 imbalance_ratio=2.076385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34943 max_load_reduction_pct=0.513573 moved_to_replicas=67595 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[72691, 50187, 32733, 31683, 43, 69, 74714, 24] avg=32768.0000 max=74714 max_rank=6 min=24 min_rank=7 imbalance_ratio=2.280090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34943 max_load_reduction_pct=0.513573 moved_to_replicas=67595 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32104, 65748, 29780, 22843, 68039, 32753, 10607, 270] avg=32768.0000 max=68039 max_rank=4 min=270 min_rank=7 imbalance_ratio=2.076385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32104, 65748, 29780, 22843, 68039, 32753, 10607, 270] avg=32768.0000 max=68039 max_rank=4 min=270 min_rank=7 imbalance_ratio=2.076385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34943 max_load_reduction_pct=0.513573 moved_to_replicas=67595 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34943 max_load_reduction_pct=0.513573 moved_to_replicas=67595 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[24820, 65228, 42109, 6868, 32760, 56896, 35, 33428] avg=32768.0000 max=65228 max_rank=1 min=35 min_rank=6 imbalance_ratio=1.990601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] avg=32768.0000 max=33097 max_rank=0 min=31683 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41617 max_load_reduction_pct=0.557017 moved_to_replicas=98304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[72691, 50187, 32733, 31683, 43, 69, 74714, 24] avg=32768.0000 max=74714 max_rank=6 min=24 min_rank=7 imbalance_ratio=2.280090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34943 max_load_reduction_pct=0.513573 moved_to_replicas=67595 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32104, 33096, 33096, 32139, 33096, 32753, 33096, 32764] avg=32768.0000 max=33096 max_rank=1 min=32104 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=34943 max_load_reduction_pct=0.513573 moved_to_replicas=67595 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[72691, 50187, 32733, 31683, 43, 69, 74714, 24] avg=32768.0000 max=74714 max_rank=6 min=24 min_rank=7 imbalance_ratio=2.280090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] avg=32768.0000 max=33097 max_rank=5 min=31736 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.492595 moved_to_replicas=65277 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[72691, 50187, 32733, 31683, 43, 69, 74714, 24] avg=32768.0000 max=74714 max_rank=6 min=24 min_rank=7 imbalance_ratio=2.280090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[72691, 50187, 32733, 31683, 43, 69, 74714, 24] avg=32768.0000 max=74714 max_rank=6 min=24 min_rank=7 imbalance_ratio=2.280090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] avg=32768.0000 max=33097 max_rank=0 min=31683 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41617 max_load_reduction_pct=0.557017 moved_to_replicas=98304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[24820, 65228, 42109, 6868, 32760, 56896, 35, 33428] avg=32768.0000 max=65228 max_rank=1 min=35 min_rank=6 imbalance_ratio=1.990601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] avg=32768.0000 max=33097 max_rank=0 min=31683 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41617 max_load_reduction_pct=0.557017 moved_to_replicas=98304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] avg=32768.0000 max=33097 max_rank=0 min=31683 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41617 max_load_reduction_pct=0.557017 moved_to_replicas=98304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[72691, 50187, 32733, 31683, 43, 69, 74714, 24] avg=32768.0000 max=74714 max_rank=6 min=24 min_rank=7 imbalance_ratio=2.280090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[72691, 50187, 32733, 31683, 43, 69, 74714, 24] avg=32768.0000 max=74714 max_rank=6 min=24 min_rank=7 imbalance_ratio=2.280090 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] avg=32768.0000 max=33097 max_rank=0 min=31683 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41617 max_load_reduction_pct=0.557017 moved_to_replicas=98304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[30455, 39392, 33842, 45702, 15151, 31498, 27131, 38973] avg=32768.0000 max=45702 max_rank=3 min=15151 min_rank=4 imbalance_ratio=1.394714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] avg=32768.0000 max=33097 max_rank=5 min=31736 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.492595 moved_to_replicas=65277 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[24820, 65228, 42109, 6868, 32760, 56896, 35, 33428] avg=32768.0000 max=65228 max_rank=1 min=35 min_rank=6 imbalance_ratio=1.990601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] avg=32768.0000 max=33097 max_rank=0 min=31683 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41617 max_load_reduction_pct=0.557017 moved_to_replicas=98304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[33097, 33097, 32733, 31683, 33094, 32787, 33096, 32557] avg=32768.0000 max=33097 max_rank=0 min=31683 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=41617 max_load_reduction_pct=0.557017 moved_to_replicas=98304 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31498 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12605 max_load_reduction_pct=0.275808 moved_to_replicas=25525 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[24820, 65228, 42109, 6868, 32760, 56896, 35, 33428] avg=32768.0000 max=65228 max_rank=1 min=35 min_rank=6 imbalance_ratio=1.990601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[24820, 65228, 42109, 6868, 32760, 56896, 35, 33428] avg=32768.0000 max=65228 max_rank=1 min=35 min_rank=6 imbalance_ratio=1.990601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[24820, 65228, 42109, 6868, 32760, 56896, 35, 33428] avg=32768.0000 max=65228 max_rank=1 min=35 min_rank=6 imbalance_ratio=1.990601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] avg=32768.0000 max=33097 max_rank=5 min=31736 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.492595 moved_to_replicas=65277 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[30455, 39392, 33842, 45702, 15151, 31498, 27131, 38973] avg=32768.0000 max=45702 max_rank=3 min=15151 min_rank=4 imbalance_ratio=1.394714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] avg=32768.0000 max=33097 max_rank=5 min=31736 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.492595 moved_to_replicas=65277 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] avg=32768.0000 max=33097 max_rank=5 min=31736 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.492595 moved_to_replicas=65277 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[24820, 65228, 42109, 6868, 32760, 56896, 35, 33428] avg=32768.0000 max=65228 max_rank=1 min=35 min_rank=6 imbalance_ratio=1.990601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[24820, 65228, 42109, 6868, 32760, 56896, 35, 33428] avg=32768.0000 max=65228 max_rank=1 min=35 min_rank=6 imbalance_ratio=1.990601 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[36491, 32656, 6645, 7166, 100676, 54036, 558, 23916] avg=32768.0000 max=100676 max_rank=4 min=558 min_rank=6 imbalance_ratio=3.072388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31498 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12605 max_load_reduction_pct=0.275808 moved_to_replicas=25525 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[30455, 39392, 33842, 45702, 15151, 31498, 27131, 38973] avg=32768.0000 max=45702 max_rank=3 min=15151 min_rank=4 imbalance_ratio=1.394714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] avg=32768.0000 max=33097 max_rank=5 min=31736 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.492595 moved_to_replicas=65277 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] avg=32768.0000 max=33097 max_rank=5 min=31736 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.492595 moved_to_replicas=65277 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 31736, 32760, 33097, 32167, 33096] avg=32768.0000 max=33097 max_rank=5 min=31736 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=32131 max_load_reduction_pct=0.492595 moved_to_replicas=65277 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] avg=32768.0000 max=33097 max_rank=4 min=30912 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67579 max_load_reduction_pct=0.671252 moved_to_replicas=91915 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[30455, 39392, 33842, 45702, 15151, 31498, 27131, 38973] avg=32768.0000 max=45702 max_rank=3 min=15151 min_rank=4 imbalance_ratio=1.394714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[30455, 39392, 33842, 45702, 15151, 31498, 27131, 38973] avg=32768.0000 max=45702 max_rank=3 min=15151 min_rank=4 imbalance_ratio=1.394714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31498 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12605 max_load_reduction_pct=0.275808 moved_to_replicas=25525 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[36491, 32656, 6645, 7166, 100676, 54036, 558, 23916] avg=32768.0000 max=100676 max_rank=4 min=558 min_rank=6 imbalance_ratio=3.072388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31498 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12605 max_load_reduction_pct=0.275808 moved_to_replicas=25525 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31498 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12605 max_load_reduction_pct=0.275808 moved_to_replicas=25525 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[30455, 39392, 33842, 45702, 15151, 31498, 27131, 38973] avg=32768.0000 max=45702 max_rank=3 min=15151 min_rank=4 imbalance_ratio=1.394714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[30455, 39392, 33842, 45702, 15151, 31498, 27131, 38973] avg=32768.0000 max=45702 max_rank=3 min=15151 min_rank=4 imbalance_ratio=1.394714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[30455, 39392, 33842, 45702, 15151, 31498, 27131, 38973] avg=32768.0000 max=45702 max_rank=3 min=15151 min_rank=4 imbalance_ratio=1.394714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] avg=32768.0000 max=33097 max_rank=4 min=30912 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67579 max_load_reduction_pct=0.671252 moved_to_replicas=91915 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[36491, 32656, 6645, 7166, 100676, 54036, 558, 23916] avg=32768.0000 max=100676 max_rank=4 min=558 min_rank=6 imbalance_ratio=3.072388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31498 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12605 max_load_reduction_pct=0.275808 moved_to_replicas=25525 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31498 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12605 max_load_reduction_pct=0.275808 moved_to_replicas=25525 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[32070, 33095, 33096, 33096, 33096, 31498, 33097, 33096] avg=32768.0000 max=33097 max_rank=6 min=31498 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=12605 max_load_reduction_pct=0.275808 moved_to_replicas=25525 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[36491, 32656, 6645, 7166, 100676, 54036, 558, 23916] avg=32768.0000 max=100676 max_rank=4 min=558 min_rank=6 imbalance_ratio=3.072388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[36491, 32656, 6645, 7166, 100676, 54036, 558, 23916] avg=32768.0000 max=100676 max_rank=4 min=558 min_rank=6 imbalance_ratio=3.072388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] avg=32768.0000 max=33097 max_rank=4 min=30912 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67579 max_load_reduction_pct=0.671252 moved_to_replicas=91915 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] avg=32768.0000 max=33097 max_rank=4 min=30912 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67579 max_load_reduction_pct=0.671252 moved_to_replicas=91915 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] avg=32768.0000 max=33097 max_rank=4 min=30912 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67579 max_load_reduction_pct=0.671252 moved_to_replicas=91915 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[36491, 32656, 6645, 7166, 100676, 54036, 558, 23916] avg=32768.0000 max=100676 max_rank=4 min=558 min_rank=6 imbalance_ratio=3.072388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[36491, 32656, 6645, 7166, 100676, 54036, 558, 23916] avg=32768.0000 max=100676 max_rank=4 min=558 min_rank=6 imbalance_ratio=3.072388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[36491, 32656, 6645, 7166, 100676, 54036, 558, 23916] avg=32768.0000 max=100676 max_rank=4 min=558 min_rank=6 imbalance_ratio=3.072388 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] avg=32768.0000 max=33097 max_rank=4 min=30912 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67579 max_load_reduction_pct=0.671252 moved_to_replicas=91915 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] avg=32768.0000 max=33097 max_rank=4 min=30912 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67579 max_load_reduction_pct=0.671252 moved_to_replicas=91915 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[33096, 32656, 33096, 33095, 33097, 33096, 33096, 30912] avg=32768.0000 max=33097 max_rank=4 min=30912 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67579 max_load_reduction_pct=0.671252 moved_to_replicas=91915 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=10 total_expected_recv=327457 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=13824 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=10 total_expected_recv=323325 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=18944 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=10 total_expected_recv=323999 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16512 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=10 total_expected_recv=329642 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=11136 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=10 total_expected_recv=328945 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=12416 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=10 total_expected_recv=330624 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=10240 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=10 total_expected_recv=330123 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=11008 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=10 total_expected_recv=327325 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=14336 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 18:02:38] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 244.0 | throughput per GPU (TFLOP/s/GPU): 506.1 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.109313E+01 | loss scale: 1.0 | grad norm: 107.040 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33920 | min safe capacity: 1.0352 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=34816 expected_recv=32536 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=34816 expected_recv=31032 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=34816 expected_recv=32536 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=34816 expected_recv=31032 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=31744 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=34816 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=34816 expected_recv=32285 expected_overflow=0 exact_recv_per_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=34816 expected_recv=32805 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=34816 expected_recv=31574 expected_overflow=0 exact_recv_per_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=34816 expected_recv=32285 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=34816 expected_recv=31574 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=32384 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=34816 expected_recv=31063 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=34816 expected_recv=31063 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=34816 expected_recv=32506 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=34048 slack_slots=768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32000 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=34816 expected_recv=32506 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=34816 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=34816 expected_recv=32765 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=34816 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=34816 expected_recv=31392 expected_overflow=0 exact_recv_per_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=34816 expected_recv=32507 expected_overflow=0 exact_recv_per_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=34816 expected_recv=31392 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=34816 expected_recv=33098 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=34816 expected_recv=32507 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=34816 expected_recv=32335 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=34816 expected_recv=32335 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=34816 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=34816 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=34816 expected_recv=32865 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=34048 slack_slots=768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=34816 expected_recv=31784 expected_overflow=0 exact_recv_per_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=34816 expected_recv=32776 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=34816 expected_recv=31784 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=34816 expected_recv=31113 expected_overflow=0 exact_recv_per_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=34816 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=34816 expected_recv=31113 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=34816 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=34816 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=34816 expected_recv=32781 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=34816 expected_recv=31509 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=34816 expected_recv=31509 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=34816 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=34816 expected_recv=32732 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=34816 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=34816 expected_recv=32751 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=34816 expected_recv=31727 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=34816 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=34816 expected_recv=31727 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32384 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=34816 expected_recv=32853 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=34816 expected_recv=32403 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=34816 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=34816 expected_recv=32403 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=34816 expected_recv=32044 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=34816 expected_recv=32779 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=34816 expected_recv=32044 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=34816 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=34816 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=34816 expected_recv=32763 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=34816 expected_recv=32830 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=34816 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=34816 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=34816 expected_recv=32771 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=34816 expected_recv=32746 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=34816 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=34816 expected_recv=32778 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=34816 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=34816 expected_recv=32772 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=34816 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[26945, 9806, 29063, 55189, 20075, 60833, 31032, 29201] avg=32768.0000 max=60833 max_rank=5 min=9806 min_rank=1 imbalance_ratio=1.856476 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=34816 expected_recv=31954 expected_overflow=0 exact_recv_per_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=34816 expected_recv=32763 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=34816 expected_recv=31954 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[26945, 9806, 29063, 55189, 20075, 60833, 31032, 29201] avg=32768.0000 max=60833 max_rank=5 min=9806 min_rank=1 imbalance_ratio=1.856476 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] avg=32768.0000 max=33097 max_rank=2 min=31032 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=27736 max_load_reduction_pct=0.455937 moved_to_replicas=49830 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[26945, 9806, 29063, 55189, 20075, 60833, 31032, 29201] avg=32768.0000 max=60833 max_rank=5 min=9806 min_rank=1 imbalance_ratio=1.856476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[26945, 9806, 29063, 55189, 20075, 60833, 31032, 29201] avg=32768.0000 max=60833 max_rank=5 min=9806 min_rank=1 imbalance_ratio=1.856476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[26945, 9806, 29063, 55189, 20075, 60833, 31032, 29201] avg=32768.0000 max=60833 max_rank=5 min=9806 min_rank=1 imbalance_ratio=1.856476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] avg=32768.0000 max=33097 max_rank=2 min=31032 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=27736 max_load_reduction_pct=0.455937 moved_to_replicas=49830 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=34816 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[26945, 9806, 29063, 55189, 20075, 60833, 31032, 29201] avg=32768.0000 max=60833 max_rank=5 min=9806 min_rank=1 imbalance_ratio=1.856476 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=34816 expected_recv=32766 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] avg=32768.0000 max=33097 max_rank=2 min=31032 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=27736 max_load_reduction_pct=0.455937 moved_to_replicas=49830 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] avg=32768.0000 max=33097 max_rank=2 min=31032 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=27736 max_load_reduction_pct=0.455937 moved_to_replicas=49830 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[94606, 5647, 22935, 52375, 1483, 15011, 2908, 67179] avg=32768.0000 max=94606 max_rank=0 min=1483 min_rank=4 imbalance_ratio=2.887146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[26945, 9806, 29063, 55189, 20075, 60833, 31032, 29201] avg=32768.0000 max=60833 max_rank=5 min=9806 min_rank=1 imbalance_ratio=1.856476 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] avg=32768.0000 max=33097 max_rank=2 min=31032 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=27736 max_load_reduction_pct=0.455937 moved_to_replicas=49830 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] avg=32768.0000 max=33097 max_rank=2 min=31032 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=27736 max_load_reduction_pct=0.455937 moved_to_replicas=49830 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[94606, 5647, 22935, 52375, 1483, 15011, 2908, 67179] avg=32768.0000 max=94606 max_rank=0 min=1483 min_rank=4 imbalance_ratio=2.887146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] avg=32768.0000 max=33096 max_rank=0 min=31574 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61510 max_load_reduction_pct=0.650170 moved_to_replicas=114872 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] avg=32768.0000 max=33097 max_rank=2 min=31032 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=27736 max_load_reduction_pct=0.455937 moved_to_replicas=49830 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[94606, 5647, 22935, 52375, 1483, 15011, 2908, 67179] avg=32768.0000 max=94606 max_rank=0 min=1483 min_rank=4 imbalance_ratio=2.887146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[94606, 5647, 22935, 52375, 1483, 15011, 2908, 67179] avg=32768.0000 max=94606 max_rank=0 min=1483 min_rank=4 imbalance_ratio=2.887146 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=34816 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[94606, 5647, 22935, 52375, 1483, 15011, 2908, 67179] avg=32768.0000 max=94606 max_rank=0 min=1483 min_rank=4 imbalance_ratio=2.887146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] avg=32768.0000 max=33096 max_rank=0 min=31574 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61510 max_load_reduction_pct=0.650170 moved_to_replicas=114872 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=34816 expected_recv=32620 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] avg=32768.0000 max=33096 max_rank=0 min=31574 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61510 max_load_reduction_pct=0.650170 moved_to_replicas=114872 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[86843, 15335, 29299, 51049, 32506, 40181, 87, 6844] avg=32768.0000 max=86843 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.650238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] avg=32768.0000 max=33096 max_rank=0 min=31574 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61510 max_load_reduction_pct=0.650170 moved_to_replicas=114872 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[94606, 5647, 22935, 52375, 1483, 15011, 2908, 67179] avg=32768.0000 max=94606 max_rank=0 min=1483 min_rank=4 imbalance_ratio=2.887146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[94606, 5647, 22935, 52375, 1483, 15011, 2908, 67179] avg=32768.0000 max=94606 max_rank=0 min=1483 min_rank=4 imbalance_ratio=2.887146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] avg=32768.0000 max=33096 max_rank=0 min=31574 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61510 max_load_reduction_pct=0.650170 moved_to_replicas=114872 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[86843, 15335, 29299, 51049, 32506, 40181, 87, 6844] avg=32768.0000 max=86843 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.650238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] avg=32768.0000 max=33096 max_rank=0 min=31063 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53747 max_load_reduction_pct=0.618898 moved_to_replicas=78785 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] avg=32768.0000 max=33096 max_rank=0 min=31574 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61510 max_load_reduction_pct=0.650170 moved_to_replicas=114872 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[86843, 15335, 29299, 51049, 32506, 40181, 87, 6844] avg=32768.0000 max=86843 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.650238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] avg=32768.0000 max=33096 max_rank=0 min=31574 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61510 max_load_reduction_pct=0.650170 moved_to_replicas=114872 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[86843, 15335, 29299, 51049, 32506, 40181, 87, 6844] avg=32768.0000 max=86843 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.650238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[26945, 9806, 29063, 55189, 20075, 60833, 31032, 29201] avg=32768.0000 max=60833 max_rank=5 min=9806 min_rank=1 imbalance_ratio=1.856476 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[86843, 15335, 29299, 51049, 32506, 40181, 87, 6844] avg=32768.0000 max=86843 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.650238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] avg=32768.0000 max=33096 max_rank=0 min=31063 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53747 max_load_reduction_pct=0.618898 moved_to_replicas=78785 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] avg=32768.0000 max=33096 max_rank=0 min=31063 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53747 max_load_reduction_pct=0.618898 moved_to_replicas=78785 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32507, 23125, 16173, 55774, 44535, 33964, 23301, 32765] avg=32768.0000 max=55774 max_rank=3 min=16173 min_rank=2 imbalance_ratio=1.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] avg=32768.0000 max=33096 max_rank=0 min=31063 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53747 max_load_reduction_pct=0.618898 moved_to_replicas=78785 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[86843, 15335, 29299, 51049, 32506, 40181, 87, 6844] avg=32768.0000 max=86843 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.650238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] avg=32768.0000 max=33096 max_rank=0 min=31063 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53747 max_load_reduction_pct=0.618898 moved_to_replicas=78785 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[33095, 33096, 33097, 33096, 32536, 33096, 31032, 33096] avg=32768.0000 max=33097 max_rank=2 min=31032 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=27736 max_load_reduction_pct=0.455937 moved_to_replicas=49830 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[86843, 15335, 29299, 51049, 32506, 40181, 87, 6844] avg=32768.0000 max=86843 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.650238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] avg=32768.0000 max=33098 max_rank=4 min=31392 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=22676 max_load_reduction_pct=0.406569 moved_to_replicas=34985 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32507, 23125, 16173, 55774, 44535, 33964, 23301, 32765] avg=32768.0000 max=55774 max_rank=3 min=16173 min_rank=2 imbalance_ratio=1.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] avg=32768.0000 max=33096 max_rank=0 min=31063 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53747 max_load_reduction_pct=0.618898 moved_to_replicas=78785 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32507, 23125, 16173, 55774, 44535, 33964, 23301, 32765] avg=32768.0000 max=55774 max_rank=3 min=16173 min_rank=2 imbalance_ratio=1.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32507, 23125, 16173, 55774, 44535, 33964, 23301, 32765] avg=32768.0000 max=55774 max_rank=3 min=16173 min_rank=2 imbalance_ratio=1.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] avg=32768.0000 max=33096 max_rank=0 min=31063 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53747 max_load_reduction_pct=0.618898 moved_to_replicas=78785 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] avg=32768.0000 max=33098 max_rank=4 min=31392 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=22676 max_load_reduction_pct=0.406569 moved_to_replicas=34985 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32507, 23125, 16173, 55774, 44535, 33964, 23301, 32765] avg=32768.0000 max=55774 max_rank=3 min=16173 min_rank=2 imbalance_ratio=1.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[94606, 5647, 22935, 52375, 1483, 15011, 2908, 67179] avg=32768.0000 max=94606 max_rank=0 min=1483 min_rank=4 imbalance_ratio=2.887146 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[67050, 26, 98691, 64402, 31784, 122, 38, 31] avg=32768.0000 max=98691 max_rank=2 min=26 min_rank=1 imbalance_ratio=3.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] avg=32768.0000 max=33098 max_rank=4 min=31392 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=22676 max_load_reduction_pct=0.406569 moved_to_replicas=34985 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] avg=32768.0000 max=33098 max_rank=4 min=31392 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=22676 max_load_reduction_pct=0.406569 moved_to_replicas=34985 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32507, 23125, 16173, 55774, 44535, 33964, 23301, 32765] avg=32768.0000 max=55774 max_rank=3 min=16173 min_rank=2 imbalance_ratio=1.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] avg=32768.0000 max=33098 max_rank=4 min=31392 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=22676 max_load_reduction_pct=0.406569 moved_to_replicas=34985 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[33096, 31574, 33096, 33096, 33096, 32285, 32805, 33096] avg=32768.0000 max=33096 max_rank=0 min=31574 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=61510 max_load_reduction_pct=0.650170 moved_to_replicas=114872 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32507, 23125, 16173, 55774, 44535, 33964, 23301, 32765] avg=32768.0000 max=55774 max_rank=3 min=16173 min_rank=2 imbalance_ratio=1.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] avg=32768.0000 max=33096 max_rank=0 min=31784 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65595 max_load_reduction_pct=0.664650 moved_to_replicas=130855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[67050, 26, 98691, 64402, 31784, 122, 38, 31] avg=32768.0000 max=98691 max_rank=2 min=26 min_rank=1 imbalance_ratio=3.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[67050, 26, 98691, 64402, 31784, 122, 38, 31] avg=32768.0000 max=98691 max_rank=2 min=26 min_rank=1 imbalance_ratio=3.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] avg=32768.0000 max=33098 max_rank=4 min=31392 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=22676 max_load_reduction_pct=0.406569 moved_to_replicas=34985 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[67050, 26, 98691, 64402, 31784, 122, 38, 31] avg=32768.0000 max=98691 max_rank=2 min=26 min_rank=1 imbalance_ratio=3.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] avg=32768.0000 max=33098 max_rank=4 min=31392 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=22676 max_load_reduction_pct=0.406569 moved_to_replicas=34985 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] avg=32768.0000 max=33096 max_rank=0 min=31784 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65595 max_load_reduction_pct=0.664650 moved_to_replicas=130855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[67050, 26, 98691, 64402, 31784, 122, 38, 31] avg=32768.0000 max=98691 max_rank=2 min=26 min_rank=1 imbalance_ratio=3.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[44, 5859, 33654, 75778, 35173, 32770, 16000, 62866] avg=32768.0000 max=75778 max_rank=3 min=44 min_rank=0 imbalance_ratio=2.312561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[86843, 15335, 29299, 51049, 32506, 40181, 87, 6844] avg=32768.0000 max=86843 max_rank=0 min=87 min_rank=6 imbalance_ratio=2.650238 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] avg=32768.0000 max=33096 max_rank=0 min=31784 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65595 max_load_reduction_pct=0.664650 moved_to_replicas=130855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] avg=32768.0000 max=33096 max_rank=0 min=31784 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65595 max_load_reduction_pct=0.664650 moved_to_replicas=130855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] avg=32768.0000 max=33096 max_rank=0 min=31784 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65595 max_load_reduction_pct=0.664650 moved_to_replicas=130855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[67050, 26, 98691, 64402, 31784, 122, 38, 31] avg=32768.0000 max=98691 max_rank=2 min=26 min_rank=1 imbalance_ratio=3.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] avg=32768.0000 max=33097 max_rank=1 min=31113 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42681 max_load_reduction_pct=0.563237 moved_to_replicas=75087 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[67050, 26, 98691, 64402, 31784, 122, 38, 31] avg=32768.0000 max=98691 max_rank=2 min=26 min_rank=1 imbalance_ratio=3.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33095, 32506, 33096, 31063, 33096] avg=32768.0000 max=33096 max_rank=0 min=31063 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=53747 max_load_reduction_pct=0.618898 moved_to_replicas=78785 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[44, 5859, 33654, 75778, 35173, 32770, 16000, 62866] avg=32768.0000 max=75778 max_rank=3 min=44 min_rank=0 imbalance_ratio=2.312561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[44, 5859, 33654, 75778, 35173, 32770, 16000, 62866] avg=32768.0000 max=75778 max_rank=3 min=44 min_rank=0 imbalance_ratio=2.312561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] avg=32768.0000 max=33096 max_rank=0 min=31784 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65595 max_load_reduction_pct=0.664650 moved_to_replicas=130855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[44, 5859, 33654, 75778, 35173, 32770, 16000, 62866] avg=32768.0000 max=75778 max_rank=3 min=44 min_rank=0 imbalance_ratio=2.312561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] avg=32768.0000 max=33096 max_rank=0 min=31784 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65595 max_load_reduction_pct=0.664650 moved_to_replicas=130855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] avg=32768.0000 max=33097 max_rank=1 min=31113 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42681 max_load_reduction_pct=0.563237 moved_to_replicas=75087 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[44, 5859, 33654, 75778, 35173, 32770, 16000, 62866] avg=32768.0000 max=75778 max_rank=3 min=44 min_rank=0 imbalance_ratio=2.312561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[65335, 32768, 32732, 28950, 5, 501, 70538, 31315] avg=32768.0000 max=70538 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.152649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32507, 23125, 16173, 55774, 44535, 33964, 23301, 32765] avg=32768.0000 max=55774 max_rank=3 min=16173 min_rank=2 imbalance_ratio=1.702087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] avg=32768.0000 max=33097 max_rank=1 min=31113 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42681 max_load_reduction_pct=0.563237 moved_to_replicas=75087 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] avg=32768.0000 max=33097 max_rank=1 min=31113 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42681 max_load_reduction_pct=0.563237 moved_to_replicas=75087 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] avg=32768.0000 max=33097 max_rank=1 min=31113 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42681 max_load_reduction_pct=0.563237 moved_to_replicas=75087 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[44, 5859, 33654, 75778, 35173, 32770, 16000, 62866] avg=32768.0000 max=75778 max_rank=3 min=44 min_rank=0 imbalance_ratio=2.312561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] avg=32768.0000 max=33097 max_rank=3 min=31509 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37441 max_load_reduction_pct=0.530792 moved_to_replicas=69681 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[44, 5859, 33654, 75778, 35173, 32770, 16000, 62866] avg=32768.0000 max=75778 max_rank=3 min=44 min_rank=0 imbalance_ratio=2.312561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32507, 31392, 33095, 33097, 33098, 33096, 33094, 32765] avg=32768.0000 max=33098 max_rank=4 min=31392 min_rank=1 imbalance_ratio=1.010071 max_load_reduction=22676 max_load_reduction_pct=0.406569 moved_to_replicas=34985 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[65335, 32768, 32732, 28950, 5, 501, 70538, 31315] avg=32768.0000 max=70538 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.152649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[65335, 32768, 32732, 28950, 5, 501, 70538, 31315] avg=32768.0000 max=70538 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.152649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] avg=32768.0000 max=33097 max_rank=1 min=31113 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42681 max_load_reduction_pct=0.563237 moved_to_replicas=75087 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[65335, 32768, 32732, 28950, 5, 501, 70538, 31315] avg=32768.0000 max=70538 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.152649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] avg=32768.0000 max=33097 max_rank=1 min=31113 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42681 max_load_reduction_pct=0.563237 moved_to_replicas=75087 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[65335, 32768, 32732, 28950, 5, 501, 70538, 31315] avg=32768.0000 max=70538 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.152649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] avg=32768.0000 max=33097 max_rank=3 min=31509 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37441 max_load_reduction_pct=0.530792 moved_to_replicas=69681 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[65490, 37251, 32853, 9, 32779, 38714, 10916, 44132] avg=32768.0000 max=65490 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[67050, 26, 98691, 64402, 31784, 122, 38, 31] avg=32768.0000 max=98691 max_rank=2 min=26 min_rank=1 imbalance_ratio=3.011810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] avg=32768.0000 max=33097 max_rank=3 min=31509 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37441 max_load_reduction_pct=0.530792 moved_to_replicas=69681 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] avg=32768.0000 max=33097 max_rank=3 min=31509 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37441 max_load_reduction_pct=0.530792 moved_to_replicas=69681 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] avg=32768.0000 max=33097 max_rank=3 min=31509 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37441 max_load_reduction_pct=0.530792 moved_to_replicas=69681 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] avg=32768.0000 max=33096 max_rank=0 min=31727 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=53203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[33096, 32776, 33096, 33096, 31784, 32865, 33096, 32335] avg=32768.0000 max=33096 max_rank=0 min=31784 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=65595 max_load_reduction_pct=0.664650 moved_to_replicas=130855 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[65335, 32768, 32732, 28950, 5, 501, 70538, 31315] avg=32768.0000 max=70538 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.152649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[65335, 32768, 32732, 28950, 5, 501, 70538, 31315] avg=32768.0000 max=70538 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.152649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[65490, 37251, 32853, 9, 32779, 38714, 10916, 44132] avg=32768.0000 max=65490 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[65490, 37251, 32853, 9, 32779, 38714, 10916, 44132] avg=32768.0000 max=65490 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[65490, 37251, 32853, 9, 32779, 38714, 10916, 44132] avg=32768.0000 max=65490 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[65490, 37251, 32853, 9, 32779, 38714, 10916, 44132] avg=32768.0000 max=65490 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32771, 816, 32763, 34791, 17, 84, 30349, 130553] avg=32768.0000 max=130553 max_rank=7 min=17 min_rank=4 imbalance_ratio=3.984161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] avg=32768.0000 max=33096 max_rank=0 min=31727 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=53203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] avg=32768.0000 max=33097 max_rank=3 min=31509 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37441 max_load_reduction_pct=0.530792 moved_to_replicas=69681 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] avg=32768.0000 max=33097 max_rank=3 min=31509 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37441 max_load_reduction_pct=0.530792 moved_to_replicas=69681 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[44, 5859, 33654, 75778, 35173, 32770, 16000, 62866] avg=32768.0000 max=75778 max_rank=3 min=44 min_rank=0 imbalance_ratio=2.312561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] avg=32768.0000 max=33096 max_rank=0 min=31727 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=53203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] avg=32768.0000 max=33096 max_rank=0 min=31727 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=53203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] avg=32768.0000 max=33096 max_rank=0 min=31727 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=53203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] avg=32768.0000 max=33096 max_rank=3 min=32044 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97457 max_load_reduction_pct=0.746494 moved_to_replicas=99152 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32781, 33097, 33096, 33095, 33096, 32770, 31113, 33096] avg=32768.0000 max=33097 max_rank=1 min=31113 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=42681 max_load_reduction_pct=0.563237 moved_to_replicas=75087 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32771, 816, 32763, 34791, 17, 84, 30349, 130553] avg=32768.0000 max=130553 max_rank=7 min=17 min_rank=4 imbalance_ratio=3.984161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[65490, 37251, 32853, 9, 32779, 38714, 10916, 44132] avg=32768.0000 max=65490 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[65490, 37251, 32853, 9, 32779, 38714, 10916, 44132] avg=32768.0000 max=65490 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32771, 816, 32763, 34791, 17, 84, 30349, 130553] avg=32768.0000 max=130553 max_rank=7 min=17 min_rank=4 imbalance_ratio=3.984161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32771, 816, 32763, 34791, 17, 84, 30349, 130553] avg=32768.0000 max=130553 max_rank=7 min=17 min_rank=4 imbalance_ratio=3.984161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] avg=32768.0000 max=33096 max_rank=3 min=32044 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97457 max_load_reduction_pct=0.746494 moved_to_replicas=99152 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[65707, 9, 32772, 32763, 22708, 65062, 10377, 32746] avg=32768.0000 max=65707 max_rank=0 min=9 min_rank=1 imbalance_ratio=2.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32771, 816, 32763, 34791, 17, 84, 30349, 130553] avg=32768.0000 max=130553 max_rank=7 min=17 min_rank=4 imbalance_ratio=3.984161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] avg=32768.0000 max=33096 max_rank=0 min=31727 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=53203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[65335, 32768, 32732, 28950, 5, 501, 70538, 31315] avg=32768.0000 max=70538 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.152649 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] avg=32768.0000 max=33096 max_rank=0 min=31727 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=53203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] avg=32768.0000 max=33096 max_rank=3 min=32044 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97457 max_load_reduction_pct=0.746494 moved_to_replicas=99152 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] avg=32768.0000 max=33096 max_rank=3 min=32044 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97457 max_load_reduction_pct=0.746494 moved_to_replicas=99152 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] avg=32768.0000 max=33097 max_rank=6 min=31954 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32610 max_load_reduction_pct=0.496294 moved_to_replicas=64577 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] avg=32768.0000 max=33096 max_rank=3 min=32044 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97457 max_load_reduction_pct=0.746494 moved_to_replicas=99152 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[65707, 9, 32772, 32763, 22708, 65062, 10377, 32746] avg=32768.0000 max=65707 max_rank=0 min=9 min_rank=1 imbalance_ratio=2.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[33096, 32768, 32732, 33097, 32751, 33096, 33095, 31509] avg=32768.0000 max=33097 max_rank=3 min=31509 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=37441 max_load_reduction_pct=0.530792 moved_to_replicas=69681 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32771, 816, 32763, 34791, 17, 84, 30349, 130553] avg=32768.0000 max=130553 max_rank=7 min=17 min_rank=4 imbalance_ratio=3.984161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[65707, 9, 32772, 32763, 22708, 65062, 10377, 32746] avg=32768.0000 max=65707 max_rank=0 min=9 min_rank=1 imbalance_ratio=2.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[65707, 9, 32772, 32763, 22708, 65062, 10377, 32746] avg=32768.0000 max=65707 max_rank=0 min=9 min_rank=1 imbalance_ratio=2.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32771, 816, 32763, 34791, 17, 84, 30349, 130553] avg=32768.0000 max=130553 max_rank=7 min=17 min_rank=4 imbalance_ratio=3.984161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] avg=32768.0000 max=33097 max_rank=6 min=31954 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32610 max_load_reduction_pct=0.496294 moved_to_replicas=64577 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[65707, 9, 32772, 32763, 22708, 65062, 10377, 32746] avg=32768.0000 max=65707 max_rank=0 min=9 min_rank=1 imbalance_ratio=2.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] avg=32768.0000 max=33096 max_rank=3 min=32044 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97457 max_load_reduction_pct=0.746494 moved_to_replicas=99152 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[65490, 37251, 32853, 9, 32779, 38714, 10916, 44132] avg=32768.0000 max=65490 max_rank=0 min=9 min_rank=3 imbalance_ratio=1.998596 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] avg=32768.0000 max=33097 max_rank=6 min=31954 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32610 max_load_reduction_pct=0.496294 moved_to_replicas=64577 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] avg=32768.0000 max=33097 max_rank=6 min=31954 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32610 max_load_reduction_pct=0.496294 moved_to_replicas=64577 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] avg=32768.0000 max=33096 max_rank=3 min=32044 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97457 max_load_reduction_pct=0.746494 moved_to_replicas=99152 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] avg=32768.0000 max=33097 max_rank=6 min=31954 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32610 max_load_reduction_pct=0.496294 moved_to_replicas=64577 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[33096, 33095, 32853, 32403, 32779, 33096, 31727, 33095] avg=32768.0000 max=33096 max_rank=0 min=31727 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32394 max_load_reduction_pct=0.494640 moved_to_replicas=53203 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[65707, 9, 32772, 32763, 22708, 65062, 10377, 32746] avg=32768.0000 max=65707 max_rank=0 min=9 min_rank=1 imbalance_ratio=2.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[65707, 9, 32772, 32763, 22708, 65062, 10377, 32746] avg=32768.0000 max=65707 max_rank=0 min=9 min_rank=1 imbalance_ratio=2.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32771, 816, 32763, 34791, 17, 84, 30349, 130553] avg=32768.0000 max=130553 max_rank=7 min=17 min_rank=4 imbalance_ratio=3.984161 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] avg=32768.0000 max=33097 max_rank=6 min=31954 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32610 max_load_reduction_pct=0.496294 moved_to_replicas=64577 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] avg=32768.0000 max=33097 max_rank=6 min=31954 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32610 max_load_reduction_pct=0.496294 moved_to_replicas=64577 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32771, 32766, 32763, 33096, 32778, 32830, 32044, 33096] avg=32768.0000 max=33096 max_rank=3 min=32044 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=97457 max_load_reduction_pct=0.746494 moved_to_replicas=99152 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=10 total_expected_recv=327930 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=13184 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[65707, 9, 32772, 32763, 22708, 65062, 10377, 32746] avg=32768.0000 max=65707 max_rank=0 min=9 min_rank=1 imbalance_ratio=2.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=10 total_expected_recv=322166 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=18560 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[33096, 32620, 32772, 32763, 31954, 33096, 33097, 32746] avg=32768.0000 max=33097 max_rank=6 min=31954 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=32610 max_load_reduction_pct=0.496294 moved_to_replicas=64577 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=10 total_expected_recv=329326 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=13056 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=10 total_expected_recv=329696 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=12288 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=10 total_expected_recv=329730 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=12288 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=10 total_expected_recv=329934 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=12544 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=10 total_expected_recv=326378 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=14848 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=10 total_expected_recv=326280 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=15488 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 18:02:38] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 243.4 | throughput per GPU (TFLOP/s/GPU): 507.3 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.000555E+01 | loss scale: 1.0 | grad norm: 26.611 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=34816 expected_recv=32503 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=34816 expected_recv=31065 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=34816 expected_recv=32503 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=34816 expected_recv=31065 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=31616 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=34816 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=34816 expected_recv=31128 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=34816 expected_recv=32440 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=34816 expected_recv=31128 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=31872 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=34816 expected_recv=31127 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=34816 expected_recv=31127 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=31872 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=34816 expected_recv=32439 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=34816 expected_recv=32439 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=34048 slack_slots=768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=34048 slack_slots=768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=34816 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=34816 expected_recv=33047 expected_overflow=0 exact_recv_per_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=34816 expected_recv=32546 expected_overflow=0 exact_recv_per_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=34816 expected_recv=32769 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=34816 expected_recv=33047 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=34816 expected_recv=32546 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=34816 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=34816 expected_recv=31937 expected_overflow=0 exact_recv_per_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=34816 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=34816 expected_recv=32788 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=34816 expected_recv=31937 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=34816 expected_recv=32865 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=32384 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=34816 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=34816 expected_recv=33039 expected_overflow=0 exact_recv_per_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=34816 expected_recv=31708 expected_overflow=0 exact_recv_per_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=34816 expected_recv=32757 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=34816 expected_recv=33039 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=34816 expected_recv=31708 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=34816 expected_recv=32515 expected_overflow=0 exact_recv_per_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=34816 expected_recv=32837 expected_overflow=0 exact_recv_per_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=34816 expected_recv=32515 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=34816 expected_recv=32837 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=34816 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=34816 expected_recv=32766 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=34816 expected_recv=32756 expected_overflow=0 exact_recv_per_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=34816 expected_recv=31141 expected_overflow=0 exact_recv_per_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=34816 expected_recv=32756 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=34816 expected_recv=31141 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=31616 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=34816 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=34816 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=34816 expected_recv=32785 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=34816 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=34816 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=34816 expected_recv=32166 expected_overflow=0 exact_recv_per_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=34816 expected_recv=31998 expected_overflow=0 exact_recv_per_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=34816 expected_recv=32811 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=34816 expected_recv=32166 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=34816 expected_recv=31998 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=34816 expected_recv=33100 expected_overflow=0 exact_recv_per_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=34816 expected_recv=31371 expected_overflow=0 exact_recv_per_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=34816 expected_recv=33100 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=34816 expected_recv=31371 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=31872 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=34816 expected_recv=32195 expected_overflow=0 exact_recv_per_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=34816 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=34816 expected_recv=32195 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=34816 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=34816 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=34816 expected_recv=32382 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=34816 expected_recv=32771 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=34816 expected_recv=32382 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=34816 expected_recv=32553 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=34816 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=34816 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=34816 expected_recv=32553 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=34816 expected_recv=32431 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=34816 expected_recv=32718 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=34816 expected_recv=31818 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=34816 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=34816 expected_recv=31818 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=34816 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=34816 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=34816 expected_recv=32442 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=34816 expected_recv=32403 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[25482, 15421, 29343, 52786, 19200, 59383, 28026, 32503] avg=32768.0000 max=59383 max_rank=5 min=15421 min_rank=1 imbalance_ratio=1.812225 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=34816 expected_recv=32403 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[25482, 15421, 29343, 52786, 19200, 59383, 28026, 32503] avg=32768.0000 max=59383 max_rank=5 min=15421 min_rank=1 imbalance_ratio=1.812225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[25482, 15421, 29343, 52786, 19200, 59383, 28026, 32503] avg=32768.0000 max=59383 max_rank=5 min=15421 min_rank=1 imbalance_ratio=1.812225 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[25482, 15421, 29343, 52786, 19200, 59383, 28026, 32503] avg=32768.0000 max=59383 max_rank=5 min=15421 min_rank=1 imbalance_ratio=1.812225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] avg=32768.0000 max=33097 max_rank=0 min=31065 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26286 max_load_reduction_pct=0.442652 moved_to_replicas=45977 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] avg=32768.0000 max=33097 max_rank=0 min=31065 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26286 max_load_reduction_pct=0.442652 moved_to_replicas=45977 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] avg=32768.0000 max=33097 max_rank=0 min=31065 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26286 max_load_reduction_pct=0.442652 moved_to_replicas=45977 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] avg=32768.0000 max=33097 max_rank=0 min=31065 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26286 max_load_reduction_pct=0.442652 moved_to_replicas=45977 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[25482, 15421, 29343, 52786, 19200, 59383, 28026, 32503] avg=32768.0000 max=59383 max_rank=5 min=15421 min_rank=1 imbalance_ratio=1.812225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[25482, 15421, 29343, 52786, 19200, 59383, 28026, 32503] avg=32768.0000 max=59383 max_rank=5 min=15421 min_rank=1 imbalance_ratio=1.812225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[25482, 15421, 29343, 52786, 19200, 59383, 28026, 32503] avg=32768.0000 max=59383 max_rank=5 min=15421 min_rank=1 imbalance_ratio=1.812225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[25482, 15421, 29343, 52786, 19200, 59383, 28026, 32503] avg=32768.0000 max=59383 max_rank=5 min=15421 min_rank=1 imbalance_ratio=1.812225 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[64443, 1692, 102421, 35182, 300, 1624, 46651, 9831] avg=32768.0000 max=102421 max_rank=2 min=300 min_rank=4 imbalance_ratio=3.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[64443, 1692, 102421, 35182, 300, 1624, 46651, 9831] avg=32768.0000 max=102421 max_rank=2 min=300 min_rank=4 imbalance_ratio=3.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] avg=32768.0000 max=33097 max_rank=0 min=31065 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26286 max_load_reduction_pct=0.442652 moved_to_replicas=45977 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] avg=32768.0000 max=33097 max_rank=0 min=31065 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26286 max_load_reduction_pct=0.442652 moved_to_replicas=45977 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[64443, 1692, 102421, 35182, 300, 1624, 46651, 9831] avg=32768.0000 max=102421 max_rank=2 min=300 min_rank=4 imbalance_ratio=3.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] avg=32768.0000 max=33097 max_rank=0 min=31065 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26286 max_load_reduction_pct=0.442652 moved_to_replicas=45977 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[64443, 1692, 102421, 35182, 300, 1624, 46651, 9831] avg=32768.0000 max=102421 max_rank=2 min=300 min_rank=4 imbalance_ratio=3.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[33097, 33096, 31065, 33096, 33096, 33095, 33096, 32503] avg=32768.0000 max=33097 max_rank=0 min=31065 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=26286 max_load_reduction_pct=0.442652 moved_to_replicas=45977 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31128 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69324 max_load_reduction_pct=0.676853 moved_to_replicas=116313 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31128 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69324 max_load_reduction_pct=0.676853 moved_to_replicas=116313 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31128 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69324 max_load_reduction_pct=0.676853 moved_to_replicas=116313 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31128 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69324 max_load_reduction_pct=0.676853 moved_to_replicas=116313 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[64443, 1692, 102421, 35182, 300, 1624, 46651, 9831] avg=32768.0000 max=102421 max_rank=2 min=300 min_rank=4 imbalance_ratio=3.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[64443, 1692, 102421, 35182, 300, 1624, 46651, 9831] avg=32768.0000 max=102421 max_rank=2 min=300 min_rank=4 imbalance_ratio=3.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[64443, 1692, 102421, 35182, 300, 1624, 46651, 9831] avg=32768.0000 max=102421 max_rank=2 min=300 min_rank=4 imbalance_ratio=3.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[1315, 80249, 32439, 6468, 20917, 58745, 48061, 13950] avg=32768.0000 max=80249 max_rank=1 min=1315 min_rank=0 imbalance_ratio=2.449005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[1315, 80249, 32439, 6468, 20917, 58745, 48061, 13950] avg=32768.0000 max=80249 max_rank=1 min=1315 min_rank=0 imbalance_ratio=2.449005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[64443, 1692, 102421, 35182, 300, 1624, 46651, 9831] avg=32768.0000 max=102421 max_rank=2 min=300 min_rank=4 imbalance_ratio=3.125641 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31128 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69324 max_load_reduction_pct=0.676853 moved_to_replicas=116313 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[1315, 80249, 32439, 6468, 20917, 58745, 48061, 13950] avg=32768.0000 max=80249 max_rank=1 min=1315 min_rank=0 imbalance_ratio=2.449005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31128 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69324 max_load_reduction_pct=0.676853 moved_to_replicas=116313 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31128 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69324 max_load_reduction_pct=0.676853 moved_to_replicas=116313 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[1315, 80249, 32439, 6468, 20917, 58745, 48061, 13950] avg=32768.0000 max=80249 max_rank=1 min=1315 min_rank=0 imbalance_ratio=2.449005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31127 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=47152 max_load_reduction_pct=0.587571 moved_to_replicas=87767 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31127 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=47152 max_load_reduction_pct=0.587571 moved_to_replicas=87767 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[33096, 33097, 33096, 33096, 31128, 32440, 33096, 33095] avg=32768.0000 max=33097 max_rank=1 min=31128 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=69324 max_load_reduction_pct=0.676853 moved_to_replicas=116313 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31127 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=47152 max_load_reduction_pct=0.587571 moved_to_replicas=87767 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31127 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=47152 max_load_reduction_pct=0.587571 moved_to_replicas=87767 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[1315, 80249, 32439, 6468, 20917, 58745, 48061, 13950] avg=32768.0000 max=80249 max_rank=1 min=1315 min_rank=0 imbalance_ratio=2.449005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[1315, 80249, 32439, 6468, 20917, 58745, 48061, 13950] avg=32768.0000 max=80249 max_rank=1 min=1315 min_rank=0 imbalance_ratio=2.449005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[30, 117, 31937, 95011, 32769, 33047, 36687, 32546] avg=32768.0000 max=95011 max_rank=3 min=30 min_rank=0 imbalance_ratio=2.899506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[1315, 80249, 32439, 6468, 20917, 58745, 48061, 13950] avg=32768.0000 max=80249 max_rank=1 min=1315 min_rank=0 imbalance_ratio=2.449005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[30, 117, 31937, 95011, 32769, 33047, 36687, 32546] avg=32768.0000 max=95011 max_rank=3 min=30 min_rank=0 imbalance_ratio=2.899506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[1315, 80249, 32439, 6468, 20917, 58745, 48061, 13950] avg=32768.0000 max=80249 max_rank=1 min=1315 min_rank=0 imbalance_ratio=2.449005 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31127 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=47152 max_load_reduction_pct=0.587571 moved_to_replicas=87767 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[30, 117, 31937, 95011, 32769, 33047, 36687, 32546] avg=32768.0000 max=95011 max_rank=3 min=30 min_rank=0 imbalance_ratio=2.899506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31127 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=47152 max_load_reduction_pct=0.587571 moved_to_replicas=87767 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] avg=32768.0000 max=33096 max_rank=3 min=31937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61915 max_load_reduction_pct=0.651661 moved_to_replicas=65506 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[30, 117, 31937, 95011, 32769, 33047, 36687, 32546] avg=32768.0000 max=95011 max_rank=3 min=30 min_rank=0 imbalance_ratio=2.899506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31127 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=47152 max_load_reduction_pct=0.587571 moved_to_replicas=87767 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] avg=32768.0000 max=33096 max_rank=3 min=31937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61915 max_load_reduction_pct=0.651661 moved_to_replicas=65506 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[33096, 33095, 32439, 33097, 31127, 33097, 33097, 33096] avg=32768.0000 max=33097 max_rank=3 min=31127 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=47152 max_load_reduction_pct=0.587571 moved_to_replicas=87767 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] avg=32768.0000 max=33096 max_rank=3 min=31937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61915 max_load_reduction_pct=0.651661 moved_to_replicas=65506 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[30, 117, 31937, 95011, 32769, 33047, 36687, 32546] avg=32768.0000 max=95011 max_rank=3 min=30 min_rank=0 imbalance_ratio=2.899506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] avg=32768.0000 max=33096 max_rank=3 min=31937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61915 max_load_reduction_pct=0.651661 moved_to_replicas=65506 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[30, 117, 31937, 95011, 32769, 33047, 36687, 32546] avg=32768.0000 max=95011 max_rank=3 min=30 min_rank=0 imbalance_ratio=2.899506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[65565, 32515, 368, 97238, 33372, 33039, 6, 41] avg=32768.0000 max=97238 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[65565, 32515, 368, 97238, 33372, 33039, 6, 41] avg=32768.0000 max=97238 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[30, 117, 31937, 95011, 32769, 33047, 36687, 32546] avg=32768.0000 max=95011 max_rank=3 min=30 min_rank=0 imbalance_ratio=2.899506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[30, 117, 31937, 95011, 32769, 33047, 36687, 32546] avg=32768.0000 max=95011 max_rank=3 min=30 min_rank=0 imbalance_ratio=2.899506 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] avg=32768.0000 max=33096 max_rank=3 min=31937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61915 max_load_reduction_pct=0.651661 moved_to_replicas=65506 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[65565, 32515, 368, 97238, 33372, 33039, 6, 41] avg=32768.0000 max=97238 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] avg=32768.0000 max=33096 max_rank=3 min=31937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61915 max_load_reduction_pct=0.651661 moved_to_replicas=65506 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[65565, 32515, 368, 97238, 33372, 33039, 6, 41] avg=32768.0000 max=97238 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] avg=32768.0000 max=33096 max_rank=3 min=31937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61915 max_load_reduction_pct=0.651661 moved_to_replicas=65506 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[32788, 32865, 31937, 33096, 32769, 33047, 33096, 32546] avg=32768.0000 max=33096 max_rank=3 min=31937 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=61915 max_load_reduction_pct=0.651661 moved_to_replicas=65506 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[65565, 32515, 368, 97238, 33372, 33039, 6, 41] avg=32768.0000 max=97238 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32756, 22215, 132, 60385, 36260, 5, 17543, 92848] avg=32768.0000 max=92848 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.833496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[65565, 32515, 368, 97238, 33372, 33039, 6, 41] avg=32768.0000 max=97238 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32756, 22215, 132, 60385, 36260, 5, 17543, 92848] avg=32768.0000 max=92848 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.833496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[65565, 32515, 368, 97238, 33372, 33039, 6, 41] avg=32768.0000 max=97238 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[65565, 32515, 368, 97238, 33372, 33039, 6, 41] avg=32768.0000 max=97238 max_rank=3 min=6 min_rank=6 imbalance_ratio=2.967468 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32756, 22215, 132, 60385, 36260, 5, 17543, 92848] avg=32768.0000 max=92848 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.833496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31141 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=59751 max_load_reduction_pct=0.643536 moved_to_replicas=90205 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31141 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=59751 max_load_reduction_pct=0.643536 moved_to_replicas=90205 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32756, 22215, 132, 60385, 36260, 5, 17543, 92848] avg=32768.0000 max=92848 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.833496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[33096, 32515, 32837, 33096, 33096, 33039, 32757, 31708] avg=32768.0000 max=33096 max_rank=0 min=31708 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=64142 max_load_reduction_pct=0.659639 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31141 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=59751 max_load_reduction_pct=0.643536 moved_to_replicas=90205 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32756, 22215, 132, 60385, 36260, 5, 17543, 92848] avg=32768.0000 max=92848 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.833496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31141 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=59751 max_load_reduction_pct=0.643536 moved_to_replicas=90205 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[64969, 10, 31998, 32811, 1362, 61583, 36626, 32785] avg=32768.0000 max=64969 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[64969, 10, 31998, 32811, 1362, 61583, 36626, 32785] avg=32768.0000 max=64969 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32756, 22215, 132, 60385, 36260, 5, 17543, 92848] avg=32768.0000 max=92848 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.833496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32756, 22215, 132, 60385, 36260, 5, 17543, 92848] avg=32768.0000 max=92848 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.833496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31141 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=59751 max_load_reduction_pct=0.643536 moved_to_replicas=90205 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32756, 22215, 132, 60385, 36260, 5, 17543, 92848] avg=32768.0000 max=92848 max_rank=7 min=5 min_rank=5 imbalance_ratio=2.833496 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[64969, 10, 31998, 32811, 1362, 61583, 36626, 32785] avg=32768.0000 max=64969 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] avg=32768.0000 max=33098 max_rank=4 min=31998 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=31871 max_load_reduction_pct=0.490557 moved_to_replicas=63890 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] avg=32768.0000 max=33098 max_rank=4 min=31998 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=31871 max_load_reduction_pct=0.490557 moved_to_replicas=63890 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31141 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=59751 max_load_reduction_pct=0.643536 moved_to_replicas=90205 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[64969, 10, 31998, 32811, 1362, 61583, 36626, 32785] avg=32768.0000 max=64969 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31141 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=59751 max_load_reduction_pct=0.643536 moved_to_replicas=90205 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32756, 33095, 31141, 33097, 33096, 32766, 33096, 33097] avg=32768.0000 max=33097 max_rank=3 min=31141 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=59751 max_load_reduction_pct=0.643536 moved_to_replicas=90205 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] avg=32768.0000 max=33098 max_rank=4 min=31998 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=31871 max_load_reduction_pct=0.490557 moved_to_replicas=63890 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] avg=32768.0000 max=33098 max_rank=4 min=31998 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=31871 max_load_reduction_pct=0.490557 moved_to_replicas=63890 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[64969, 10, 31998, 32811, 1362, 61583, 36626, 32785] avg=32768.0000 max=64969 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[49528, 59446, 54637, 4, 33109, 65287, 40, 93] avg=32768.0000 max=65287 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.992401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[49528, 59446, 54637, 4, 33109, 65287, 40, 93] avg=32768.0000 max=65287 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.992401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[64969, 10, 31998, 32811, 1362, 61583, 36626, 32785] avg=32768.0000 max=64969 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[64969, 10, 31998, 32811, 1362, 61583, 36626, 32785] avg=32768.0000 max=64969 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] avg=32768.0000 max=33098 max_rank=4 min=31998 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=31871 max_load_reduction_pct=0.490557 moved_to_replicas=63890 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[64969, 10, 31998, 32811, 1362, 61583, 36626, 32785] avg=32768.0000 max=64969 max_rank=0 min=10 min_rank=1 imbalance_ratio=1.982697 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[49528, 59446, 54637, 4, 33109, 65287, 40, 93] avg=32768.0000 max=65287 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.992401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] avg=32768.0000 max=33100 max_rank=7 min=31371 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=32187 max_load_reduction_pct=0.493008 moved_to_replicas=96527 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] avg=32768.0000 max=33100 max_rank=7 min=31371 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=32187 max_load_reduction_pct=0.493008 moved_to_replicas=96527 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] avg=32768.0000 max=33098 max_rank=4 min=31998 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=31871 max_load_reduction_pct=0.490557 moved_to_replicas=63890 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[49528, 59446, 54637, 4, 33109, 65287, 40, 93] avg=32768.0000 max=65287 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.992401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] avg=32768.0000 max=33098 max_rank=4 min=31998 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=31871 max_load_reduction_pct=0.490557 moved_to_replicas=63890 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[33096, 32166, 31998, 32811, 33098, 33094, 33096, 32785] avg=32768.0000 max=33098 max_rank=4 min=31998 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=31871 max_load_reduction_pct=0.490557 moved_to_replicas=63890 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] avg=32768.0000 max=33100 max_rank=7 min=31371 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=32187 max_load_reduction_pct=0.493008 moved_to_replicas=96527 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[49528, 59446, 54637, 4, 33109, 65287, 40, 93] avg=32768.0000 max=65287 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.992401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] avg=32768.0000 max=33100 max_rank=7 min=31371 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=32187 max_load_reduction_pct=0.493008 moved_to_replicas=96527 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[3, 65464, 32718, 23224, 42426, 65524, 14, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[3, 65464, 32718, 23224, 42426, 65524, 14, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[49528, 59446, 54637, 4, 33109, 65287, 40, 93] avg=32768.0000 max=65287 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.992401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[49528, 59446, 54637, 4, 33109, 65287, 40, 93] avg=32768.0000 max=65287 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.992401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[49528, 59446, 54637, 4, 33109, 65287, 40, 93] avg=32768.0000 max=65287 max_rank=5 min=4 min_rank=3 imbalance_ratio=1.992401 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] avg=32768.0000 max=33100 max_rank=7 min=31371 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=32187 max_load_reduction_pct=0.493008 moved_to_replicas=96527 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] avg=32768.0000 max=33097 max_rank=4 min=32382 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=74126 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] avg=32768.0000 max=33097 max_rank=4 min=32382 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=74126 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] avg=32768.0000 max=33100 max_rank=7 min=31371 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=32187 max_load_reduction_pct=0.493008 moved_to_replicas=96527 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[3, 65464, 32718, 23224, 42426, 65524, 14, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] avg=32768.0000 max=33100 max_rank=7 min=31371 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=32187 max_load_reduction_pct=0.493008 moved_to_replicas=96527 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[3, 65464, 32718, 23224, 42426, 65524, 14, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[33094, 33098, 33094, 32195, 33096, 33096, 31371, 33100] avg=32768.0000 max=33100 max_rank=7 min=31371 min_rank=6 imbalance_ratio=1.010132 max_load_reduction=32187 max_load_reduction_pct=0.493008 moved_to_replicas=96527 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] avg=32768.0000 max=33097 max_rank=4 min=32382 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=74126 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[3, 65464, 32718, 23224, 42426, 65524, 14, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[65536, 4, 65495, 2, 55967, 59546, 15579, 15] avg=32768.0000 max=65536 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[65536, 4, 65495, 2, 55967, 59546, 15579, 15] avg=32768.0000 max=65536 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[3, 65464, 32718, 23224, 42426, 65524, 14, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] avg=32768.0000 max=33097 max_rank=4 min=32382 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=74126 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[3, 65464, 32718, 23224, 42426, 65524, 14, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[3, 65464, 32718, 23224, 42426, 65524, 14, 32771] avg=32768.0000 max=65524 max_rank=5 min=3 min_rank=0 imbalance_ratio=1.999634 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] avg=32768.0000 max=33098 max_rank=6 min=31818 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32438 max_load_reduction_pct=0.494965 moved_to_replicas=114160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] avg=32768.0000 max=33098 max_rank=6 min=31818 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32438 max_load_reduction_pct=0.494965 moved_to_replicas=114160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] avg=32768.0000 max=33097 max_rank=4 min=32382 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=74126 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] avg=32768.0000 max=33097 max_rank=4 min=32382 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=74126 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[65536, 4, 65495, 2, 55967, 59546, 15579, 15] avg=32768.0000 max=65536 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] avg=32768.0000 max=33097 max_rank=4 min=32382 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=74126 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[65536, 4, 65495, 2, 55967, 59546, 15579, 15] avg=32768.0000 max=65536 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[32431, 33096, 32718, 32553, 33097, 33096, 32382, 32771] avg=32768.0000 max=33097 max_rank=4 min=32382 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=32427 max_load_reduction_pct=0.494887 moved_to_replicas=74126 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] avg=32768.0000 max=33098 max_rank=6 min=31818 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32438 max_load_reduction_pct=0.494965 moved_to_replicas=114160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[65536, 4, 65495, 2, 55967, 59546, 15579, 15] avg=32768.0000 max=65536 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[65536, 4, 65495, 2, 55967, 59546, 15579, 15] avg=32768.0000 max=65536 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] avg=32768.0000 max=33098 max_rank=6 min=31818 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32438 max_load_reduction_pct=0.494965 moved_to_replicas=114160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[65536, 4, 65495, 2, 55967, 59546, 15579, 15] avg=32768.0000 max=65536 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[65536, 4, 65495, 2, 55967, 59546, 15579, 15] avg=32768.0000 max=65536 max_rank=0 min=2 min_rank=3 imbalance_ratio=2.000000 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] avg=32768.0000 max=33098 max_rank=6 min=31818 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32438 max_load_reduction_pct=0.494965 moved_to_replicas=114160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] avg=32768.0000 max=33098 max_rank=6 min=31818 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32438 max_load_reduction_pct=0.494965 moved_to_replicas=114160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] avg=32768.0000 max=33098 max_rank=6 min=31818 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32438 max_load_reduction_pct=0.494965 moved_to_replicas=114160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[33096, 32403, 33096, 32442, 33094, 33097, 33098, 31818] avg=32768.0000 max=33098 max_rank=6 min=31818 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=32438 max_load_reduction_pct=0.494965 moved_to_replicas=114160 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=10 total_expected_recv=329867 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=13440 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=10 total_expected_recv=326519 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=14592 peak_demand=33100 required_capacity=1.010132 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=10 total_expected_recv=328185 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=14336 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=10 total_expected_recv=326697 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16128 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=10 total_expected_recv=329646 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=12672 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=10 total_expected_recv=328579 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=14464 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=10 total_expected_recv=328526 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=13568 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=10 total_expected_recv=323421 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=19328 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 18:02:39] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 237.1 | throughput per GPU (TFLOP/s/GPU): 520.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.050099E+01 | loss scale: 1.0 | grad norm: 8.742 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=34816 expected_recv=32462 expected_overflow=0 exact_recv_per_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=34816 expected_recv=31487 expected_overflow=0 exact_recv_per_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=34816 expected_recv=32462 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=34816 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=34816 expected_recv=31487 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=34816 expected_recv=32715 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=34816 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=34816 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=34816 expected_recv=32685 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=34816 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=34816 expected_recv=31717 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=34816 expected_recv=32568 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=34816 expected_recv=32791 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=34816 expected_recv=31717 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=34816 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=34816 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=34816 expected_recv=32767 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=34816 expected_recv=32797 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=34816 expected_recv=32169 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=34816 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=34816 expected_recv=32374 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=34816 expected_recv=32169 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=34816 expected_recv=32751 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=34816 expected_recv=32374 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=34816 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=34816 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=34816 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=34816 expected_recv=31558 expected_overflow=0 exact_recv_per_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=34816 expected_recv=32773 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=34816 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=34816 expected_recv=31558 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=34816 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=34816 expected_recv=32763 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=34816 expected_recv=32666 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=34816 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=34816 expected_recv=32862 expected_overflow=0 exact_recv_per_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=34816 expected_recv=31303 expected_overflow=0 exact_recv_per_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=34816 expected_recv=32819 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=34816 expected_recv=32862 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=34816 expected_recv=31303 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=34816 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=31616 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=34816 expected_recv=32776 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=34816 expected_recv=31279 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=34816 expected_recv=31279 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=31616 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=34816 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=34816 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=34816 expected_recv=32943 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=34816 expected_recv=32770 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=34816 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=34816 expected_recv=33024 expected_overflow=0 exact_recv_per_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=34816 expected_recv=32774 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=34816 expected_recv=33024 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=34816 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=34816 expected_recv=32515 expected_overflow=0 exact_recv_per_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=34816 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=34816 expected_recv=31709 expected_overflow=0 exact_recv_per_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=34816 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=34816 expected_recv=32515 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=34816 expected_recv=31709 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=34816 expected_recv=32206 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=34816 expected_recv=32833 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32000 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=34816 expected_recv=32206 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=34816 expected_recv=32482 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=34816 expected_recv=32323 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=34816 expected_recv=32482 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=34816 expected_recv=32323 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=34816 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=34816 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=34816 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=34816 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=34816 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=34816 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=34816 expected_recv=32731 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=34816 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=34816 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=34816 expected_recv=31812 expected_overflow=0 exact_recv_per_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=34816 expected_recv=32780 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=34816 expected_recv=31812 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=34816 expected_recv=32470 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=34816 expected_recv=32470 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=34816 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=34816 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=34816 expected_recv=32398 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=34816 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=34816 expected_recv=32398 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=34816 expected_recv=32782 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[15681, 35356, 38139, 32462, 14814, 44726, 40655, 40311] avg=32768.0000 max=44726 max_rank=5 min=14814 min_rank=4 imbalance_ratio=1.364929 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[15681, 35356, 38139, 32462, 14814, 44726, 40655, 40311] avg=32768.0000 max=44726 max_rank=5 min=14814 min_rank=4 imbalance_ratio=1.364929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[15681, 35356, 38139, 32462, 14814, 44726, 40655, 40311] avg=32768.0000 max=44726 max_rank=5 min=14814 min_rank=4 imbalance_ratio=1.364929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[15681, 35356, 38139, 32462, 14814, 44726, 40655, 40311] avg=32768.0000 max=44726 max_rank=5 min=14814 min_rank=4 imbalance_ratio=1.364929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[15681, 35356, 38139, 32462, 14814, 44726, 40655, 40311] avg=32768.0000 max=44726 max_rank=5 min=14814 min_rank=4 imbalance_ratio=1.364929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31487 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=11630 max_load_reduction_pct=0.260028 moved_to_replicas=33707 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31487 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=11630 max_load_reduction_pct=0.260028 moved_to_replicas=33707 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[15681, 35356, 38139, 32462, 14814, 44726, 40655, 40311] avg=32768.0000 max=44726 max_rank=5 min=14814 min_rank=4 imbalance_ratio=1.364929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[15681, 35356, 38139, 32462, 14814, 44726, 40655, 40311] avg=32768.0000 max=44726 max_rank=5 min=14814 min_rank=4 imbalance_ratio=1.364929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[15681, 35356, 38139, 32462, 14814, 44726, 40655, 40311] avg=32768.0000 max=44726 max_rank=5 min=14814 min_rank=4 imbalance_ratio=1.364929 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31487 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=11630 max_load_reduction_pct=0.260028 moved_to_replicas=33707 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31487 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=11630 max_load_reduction_pct=0.260028 moved_to_replicas=33707 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31487 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=11630 max_load_reduction_pct=0.260028 moved_to_replicas=33707 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[793, 27487, 32791, 32685, 32568, 79123, 16686, 40011] avg=32768.0000 max=79123 max_rank=5 min=793 min_rank=0 imbalance_ratio=2.414642 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31487 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=11630 max_load_reduction_pct=0.260028 moved_to_replicas=33707 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31487 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=11630 max_load_reduction_pct=0.260028 moved_to_replicas=33707 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[793, 27487, 32791, 32685, 32568, 79123, 16686, 40011] avg=32768.0000 max=79123 max_rank=5 min=793 min_rank=0 imbalance_ratio=2.414642 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[32715, 33096, 33096, 32462, 31487, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31487 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=11630 max_load_reduction_pct=0.260028 moved_to_replicas=33707 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[793, 27487, 32791, 32685, 32568, 79123, 16686, 40011] avg=32768.0000 max=79123 max_rank=5 min=793 min_rank=0 imbalance_ratio=2.414642 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] avg=32768.0000 max=33096 max_rank=0 min=31717 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=46027 max_load_reduction_pct=0.581715 moved_to_replicas=52942 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[793, 27487, 32791, 32685, 32568, 79123, 16686, 40011] avg=32768.0000 max=79123 max_rank=5 min=793 min_rank=0 imbalance_ratio=2.414642 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[793, 27487, 32791, 32685, 32568, 79123, 16686, 40011] avg=32768.0000 max=79123 max_rank=5 min=793 min_rank=0 imbalance_ratio=2.414642 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] avg=32768.0000 max=33096 max_rank=0 min=31717 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=46027 max_load_reduction_pct=0.581715 moved_to_replicas=52942 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[793, 27487, 32791, 32685, 32568, 79123, 16686, 40011] avg=32768.0000 max=79123 max_rank=5 min=793 min_rank=0 imbalance_ratio=2.414642 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[793, 27487, 32791, 32685, 32568, 79123, 16686, 40011] avg=32768.0000 max=79123 max_rank=5 min=793 min_rank=0 imbalance_ratio=2.414642 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] avg=32768.0000 max=33096 max_rank=0 min=31717 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=46027 max_load_reduction_pct=0.581715 moved_to_replicas=52942 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[793, 27487, 32791, 32685, 32568, 79123, 16686, 40011] avg=32768.0000 max=79123 max_rank=5 min=793 min_rank=0 imbalance_ratio=2.414642 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] avg=32768.0000 max=33096 max_rank=0 min=31717 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=46027 max_load_reduction_pct=0.581715 moved_to_replicas=52942 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] avg=32768.0000 max=33096 max_rank=0 min=31717 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=46027 max_load_reduction_pct=0.581715 moved_to_replicas=52942 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[38544, 65467, 32751, 32797, 18877, 32767, 3, 40938] avg=32768.0000 max=65467 max_rank=1 min=3 min_rank=6 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] avg=32768.0000 max=33096 max_rank=0 min=31717 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=46027 max_load_reduction_pct=0.581715 moved_to_replicas=52942 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[38544, 65467, 32751, 32797, 18877, 32767, 3, 40938] avg=32768.0000 max=65467 max_rank=1 min=3 min_rank=6 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] avg=32768.0000 max=33096 max_rank=0 min=31717 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=46027 max_load_reduction_pct=0.581715 moved_to_replicas=52942 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[33096, 33096, 32791, 32685, 32568, 33095, 31717, 33096] avg=32768.0000 max=33096 max_rank=0 min=31717 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=46027 max_load_reduction_pct=0.581715 moved_to_replicas=52942 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[38544, 65467, 32751, 32797, 18877, 32767, 3, 40938] avg=32768.0000 max=65467 max_rank=1 min=3 min_rank=6 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=45661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[38544, 65467, 32751, 32797, 18877, 32767, 3, 40938] avg=32768.0000 max=65467 max_rank=1 min=3 min_rank=6 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[38544, 65467, 32751, 32797, 18877, 32767, 3, 40938] avg=32768.0000 max=65467 max_rank=1 min=3 min_rank=6 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=45661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=45661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[38544, 65467, 32751, 32797, 18877, 32767, 3, 40938] avg=32768.0000 max=65467 max_rank=1 min=3 min_rank=6 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[38544, 65467, 32751, 32797, 18877, 32767, 3, 40938] avg=32768.0000 max=65467 max_rank=1 min=3 min_rank=6 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[38544, 65467, 32751, 32797, 18877, 32767, 3, 40938] avg=32768.0000 max=65467 max_rank=1 min=3 min_rank=6 imbalance_ratio=1.997894 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=45661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 25776, 32773, 62006, 42194, 33755, 211] avg=32768.0000 max=62006 max_rank=4 min=211 min_rank=7 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=45661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 25776, 32773, 62006, 42194, 33755, 211] avg=32768.0000 max=62006 max_rank=4 min=211 min_rank=7 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=45661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=45661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[33095, 33096, 32751, 32797, 32169, 32767, 32374, 33095] avg=32768.0000 max=33096 max_rank=1 min=32169 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=32371 max_load_reduction_pct=0.494463 moved_to_replicas=45661 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 25776, 32773, 62006, 42194, 33755, 211] avg=32768.0000 max=62006 max_rank=4 min=211 min_rank=7 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] avg=32768.0000 max=33098 max_rank=5 min=31558 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28908 max_load_reduction_pct=0.466213 moved_to_replicas=38667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] avg=32768.0000 max=33098 max_rank=5 min=31558 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28908 max_load_reduction_pct=0.466213 moved_to_replicas=38667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 25776, 32773, 62006, 42194, 33755, 211] avg=32768.0000 max=62006 max_rank=4 min=211 min_rank=7 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 25776, 32773, 62006, 42194, 33755, 211] avg=32768.0000 max=62006 max_rank=4 min=211 min_rank=7 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] avg=32768.0000 max=33098 max_rank=5 min=31558 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28908 max_load_reduction_pct=0.466213 moved_to_replicas=38667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 25776, 32773, 62006, 42194, 33755, 211] avg=32768.0000 max=62006 max_rank=4 min=211 min_rank=7 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 25776, 32773, 62006, 42194, 33755, 211] avg=32768.0000 max=62006 max_rank=4 min=211 min_rank=7 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 25776, 32773, 62006, 42194, 33755, 211] avg=32768.0000 max=62006 max_rank=4 min=211 min_rank=7 imbalance_ratio=1.892273 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[32776, 25, 63465, 20545, 64374, 4914, 43183, 32862] avg=32768.0000 max=64374 max_rank=4 min=25 min_rank=1 imbalance_ratio=1.964539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] avg=32768.0000 max=33098 max_rank=5 min=31558 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28908 max_load_reduction_pct=0.466213 moved_to_replicas=38667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] avg=32768.0000 max=33098 max_rank=5 min=31558 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28908 max_load_reduction_pct=0.466213 moved_to_replicas=38667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[32776, 25, 63465, 20545, 64374, 4914, 43183, 32862] avg=32768.0000 max=64374 max_rank=4 min=25 min_rank=1 imbalance_ratio=1.964539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] avg=32768.0000 max=33098 max_rank=5 min=31558 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28908 max_load_reduction_pct=0.466213 moved_to_replicas=38667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] avg=32768.0000 max=33098 max_rank=5 min=31558 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28908 max_load_reduction_pct=0.466213 moved_to_replicas=38667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32666, 32763, 33094, 32773, 33096, 33098, 33096, 31558] avg=32768.0000 max=33098 max_rank=5 min=31558 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=28908 max_load_reduction_pct=0.466213 moved_to_replicas=38667 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] avg=32768.0000 max=33096 max_rank=2 min=31303 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31278 max_load_reduction_pct=0.485879 moved_to_replicas=71734 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[32776, 25, 63465, 20545, 64374, 4914, 43183, 32862] avg=32768.0000 max=64374 max_rank=4 min=25 min_rank=1 imbalance_ratio=1.964539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] avg=32768.0000 max=33096 max_rank=2 min=31303 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31278 max_load_reduction_pct=0.485879 moved_to_replicas=71734 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[32776, 25, 63465, 20545, 64374, 4914, 43183, 32862] avg=32768.0000 max=64374 max_rank=4 min=25 min_rank=1 imbalance_ratio=1.964539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[32776, 25, 63465, 20545, 64374, 4914, 43183, 32862] avg=32768.0000 max=64374 max_rank=4 min=25 min_rank=1 imbalance_ratio=1.964539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[32776, 25, 63465, 20545, 64374, 4914, 43183, 32862] avg=32768.0000 max=64374 max_rank=4 min=25 min_rank=1 imbalance_ratio=1.964539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[32776, 25, 63465, 20545, 64374, 4914, 43183, 32862] avg=32768.0000 max=64374 max_rank=4 min=25 min_rank=1 imbalance_ratio=1.964539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] avg=32768.0000 max=33096 max_rank=2 min=31303 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31278 max_load_reduction_pct=0.485879 moved_to_replicas=71734 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[3, 96863, 32943, 1, 66750, 2, 65536, 46] avg=32768.0000 max=96863 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.956024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[32776, 25, 63465, 20545, 64374, 4914, 43183, 32862] avg=32768.0000 max=64374 max_rank=4 min=25 min_rank=1 imbalance_ratio=1.964539 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] avg=32768.0000 max=33096 max_rank=2 min=31303 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31278 max_load_reduction_pct=0.485879 moved_to_replicas=71734 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[3, 96863, 32943, 1, 66750, 2, 65536, 46] avg=32768.0000 max=96863 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.956024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] avg=32768.0000 max=33096 max_rank=2 min=31303 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31278 max_load_reduction_pct=0.485879 moved_to_replicas=71734 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] avg=32768.0000 max=33096 max_rank=2 min=31303 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31278 max_load_reduction_pct=0.485879 moved_to_replicas=71734 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] avg=32768.0000 max=33096 max_rank=2 min=31303 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31278 max_load_reduction_pct=0.485879 moved_to_replicas=71734 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31279 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63767 max_load_reduction_pct=0.658322 moved_to_replicas=129861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[32776, 31303, 33096, 32819, 33096, 33096, 33096, 32862] avg=32768.0000 max=33096 max_rank=2 min=31303 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=31278 max_load_reduction_pct=0.485879 moved_to_replicas=71734 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[3, 96863, 32943, 1, 66750, 2, 65536, 46] avg=32768.0000 max=96863 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.956024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31279 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63767 max_load_reduction_pct=0.658322 moved_to_replicas=129861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[3, 96863, 32943, 1, 66750, 2, 65536, 46] avg=32768.0000 max=96863 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.956024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[3, 96863, 32943, 1, 66750, 2, 65536, 46] avg=32768.0000 max=96863 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.956024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[3, 96863, 32943, 1, 66750, 2, 65536, 46] avg=32768.0000 max=96863 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.956024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31279 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63767 max_load_reduction_pct=0.658322 moved_to_replicas=129861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[3, 96863, 32943, 1, 66750, 2, 65536, 46] avg=32768.0000 max=96863 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.956024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[13758, 78115, 32515, 33024, 57147, 14745, 32833, 7] avg=32768.0000 max=78115 max_rank=1 min=7 min_rank=7 imbalance_ratio=2.383881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31279 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63767 max_load_reduction_pct=0.658322 moved_to_replicas=129861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[3, 96863, 32943, 1, 66750, 2, 65536, 46] avg=32768.0000 max=96863 max_rank=1 min=1 min_rank=3 imbalance_ratio=2.956024 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[13758, 78115, 32515, 33024, 57147, 14745, 32833, 7] avg=32768.0000 max=78115 max_rank=1 min=7 min_rank=7 imbalance_ratio=2.383881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31279 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63767 max_load_reduction_pct=0.658322 moved_to_replicas=129861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31279 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63767 max_load_reduction_pct=0.658322 moved_to_replicas=129861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31279 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63767 max_load_reduction_pct=0.658322 moved_to_replicas=129861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] avg=32768.0000 max=33098 max_rank=1 min=31709 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=45017 max_load_reduction_pct=0.576291 moved_to_replicas=69070 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[13758, 78115, 32515, 33024, 57147, 14745, 32833, 7] avg=32768.0000 max=78115 max_rank=1 min=7 min_rank=7 imbalance_ratio=2.383881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[32770, 33096, 32943, 32768, 33096, 31279, 33096, 33096] avg=32768.0000 max=33096 max_rank=1 min=31279 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=63767 max_load_reduction_pct=0.658322 moved_to_replicas=129861 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] avg=32768.0000 max=33098 max_rank=1 min=31709 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=45017 max_load_reduction_pct=0.576291 moved_to_replicas=69070 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[13758, 78115, 32515, 33024, 57147, 14745, 32833, 7] avg=32768.0000 max=78115 max_rank=1 min=7 min_rank=7 imbalance_ratio=2.383881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[13758, 78115, 32515, 33024, 57147, 14745, 32833, 7] avg=32768.0000 max=78115 max_rank=1 min=7 min_rank=7 imbalance_ratio=2.383881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[13758, 78115, 32515, 33024, 57147, 14745, 32833, 7] avg=32768.0000 max=78115 max_rank=1 min=7 min_rank=7 imbalance_ratio=2.383881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] avg=32768.0000 max=33098 max_rank=1 min=31709 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=45017 max_load_reduction_pct=0.576291 moved_to_replicas=69070 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[13758, 78115, 32515, 33024, 57147, 14745, 32833, 7] avg=32768.0000 max=78115 max_rank=1 min=7 min_rank=7 imbalance_ratio=2.383881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[65396, 64927, 65578, 0, 23, 32, 32749, 33439] avg=32768.0000 max=65578 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.001282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] avg=32768.0000 max=33098 max_rank=1 min=31709 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=45017 max_load_reduction_pct=0.576291 moved_to_replicas=69070 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[65396, 64927, 65578, 0, 23, 32, 32749, 33439] avg=32768.0000 max=65578 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.001282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[13758, 78115, 32515, 33024, 57147, 14745, 32833, 7] avg=32768.0000 max=78115 max_rank=1 min=7 min_rank=7 imbalance_ratio=2.383881 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] avg=32768.0000 max=33098 max_rank=1 min=31709 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=45017 max_load_reduction_pct=0.576291 moved_to_replicas=69070 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] avg=32768.0000 max=33098 max_rank=1 min=31709 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=45017 max_load_reduction_pct=0.576291 moved_to_replicas=69070 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[65396, 64927, 65578, 0, 23, 32, 32749, 33439] avg=32768.0000 max=65578 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.001282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32482 max_load_reduction_pct=0.495319 moved_to_replicas=96956 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] avg=32768.0000 max=33098 max_rank=1 min=31709 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=45017 max_load_reduction_pct=0.576291 moved_to_replicas=69070 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32482 max_load_reduction_pct=0.495319 moved_to_replicas=96956 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[31709, 33098, 32515, 33024, 33095, 33096, 32833, 32774] avg=32768.0000 max=33098 max_rank=1 min=31709 min_rank=0 imbalance_ratio=1.010071 max_load_reduction=45017 max_load_reduction_pct=0.576291 moved_to_replicas=69070 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[65396, 64927, 65578, 0, 23, 32, 32749, 33439] avg=32768.0000 max=65578 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.001282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32482 max_load_reduction_pct=0.495319 moved_to_replicas=96956 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[65396, 64927, 65578, 0, 23, 32, 32749, 33439] avg=32768.0000 max=65578 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.001282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[65396, 64927, 65578, 0, 23, 32, 32749, 33439] avg=32768.0000 max=65578 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.001282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32780, 32731, 63945, 438, 66806, 65424] avg=32768.0000 max=66806 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.038757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[65396, 64927, 65578, 0, 23, 32, 32749, 33439] avg=32768.0000 max=65578 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.001282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32780, 32731, 63945, 438, 66806, 65424] avg=32768.0000 max=66806 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.038757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32482 max_load_reduction_pct=0.495319 moved_to_replicas=96956 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[65396, 64927, 65578, 0, 23, 32, 32749, 33439] avg=32768.0000 max=65578 max_rank=2 min=0 min_rank=3 imbalance_ratio=2.001282 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32482 max_load_reduction_pct=0.495319 moved_to_replicas=96956 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32482 max_load_reduction_pct=0.495319 moved_to_replicas=96956 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32780, 32731, 63945, 438, 66806, 65424] avg=32768.0000 max=66806 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.038757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31812 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33710 max_load_reduction_pct=0.504595 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32482 max_load_reduction_pct=0.495319 moved_to_replicas=96956 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31812 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33710 max_load_reduction_pct=0.504595 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[33096, 33096, 33096, 32482, 32323, 32206, 32749, 33096] avg=32768.0000 max=33096 max_rank=0 min=32206 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=32482 max_load_reduction_pct=0.495319 moved_to_replicas=96956 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32780, 32731, 63945, 438, 66806, 65424] avg=32768.0000 max=66806 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.038757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31812 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33710 max_load_reduction_pct=0.504595 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32780, 32731, 63945, 438, 66806, 65424] avg=32768.0000 max=66806 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.038757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32780, 32731, 63945, 438, 66806, 65424] avg=32768.0000 max=66806 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.038757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 3, 32766, 65566, 65491, 32782, 0] avg=32768.0000 max=65566 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32780, 32731, 63945, 438, 66806, 65424] avg=32768.0000 max=66806 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.038757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 3, 32766, 65566, 65491, 32782, 0] avg=32768.0000 max=65566 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31812 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33710 max_load_reduction_pct=0.504595 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[20, 0, 32780, 32731, 63945, 438, 66806, 65424] avg=32768.0000 max=66806 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.038757 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31812 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33710 max_load_reduction_pct=0.504595 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31812 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33710 max_load_reduction_pct=0.504595 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] avg=32768.0000 max=33096 max_rank=4 min=32398 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32470 max_load_reduction_pct=0.495226 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 3, 32766, 65566, 65491, 32782, 0] avg=32768.0000 max=65566 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31812 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33710 max_load_reduction_pct=0.504595 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] avg=32768.0000 max=33096 max_rank=4 min=32398 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32470 max_load_reduction_pct=0.495226 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[31812, 32767, 32780, 32731, 33096, 32766, 33096, 33096] avg=32768.0000 max=33096 max_rank=4 min=31812 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=33710 max_load_reduction_pct=0.504595 moved_to_replicas=96887 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 3, 32766, 65566, 65491, 32782, 0] avg=32768.0000 max=65566 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] avg=32768.0000 max=33096 max_rank=4 min=32398 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32470 max_load_reduction_pct=0.495226 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 3, 32766, 65566, 65491, 32782, 0] avg=32768.0000 max=65566 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 3, 32766, 65566, 65491, 32782, 0] avg=32768.0000 max=65566 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 3, 32766, 65566, 65491, 32782, 0] avg=32768.0000 max=65566 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] avg=32768.0000 max=33096 max_rank=4 min=32398 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32470 max_load_reduction_pct=0.495226 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 3, 32766, 65566, 65491, 32782, 0] avg=32768.0000 max=65566 max_rank=4 min=0 min_rank=7 imbalance_ratio=2.000916 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] avg=32768.0000 max=33096 max_rank=4 min=32398 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32470 max_load_reduction_pct=0.495226 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] avg=32768.0000 max=33096 max_rank=4 min=32398 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32470 max_load_reduction_pct=0.495226 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] avg=32768.0000 max=33096 max_rank=4 min=32398 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32470 max_load_reduction_pct=0.495226 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32768, 32768, 32398, 32766, 33096, 33096, 32782, 32470] avg=32768.0000 max=33096 max_rank=4 min=32398 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32470 max_load_reduction_pct=0.495226 moved_to_replicas=64865 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=10 total_expected_recv=327595 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=17280 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=10 total_expected_recv=328239 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16512 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=10 total_expected_recv=327122 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=17536 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=10 total_expected_recv=328179 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16768 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=10 total_expected_recv=328560 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=15872 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=10 total_expected_recv=327307 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=17536 peak_demand=33024 required_capacity=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=10 total_expected_recv=326503 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=18560 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=10 total_expected_recv=327935 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=17664 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 18:02:39] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 229.7 | throughput per GPU (TFLOP/s/GPU): 537.5 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.447182E+00 | loss scale: 1.0 | grad norm: 5.484 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=34816 expected_recv=30472 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=34816 expected_recv=30472 actual_valid_recv=31104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=31104 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=34816 expected_recv=32814 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=34816 expected_recv=32995 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=34816 expected_recv=32814 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=34816 expected_recv=32995 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=34816 expected_recv=31545 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=34816 expected_recv=32406 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=34816 expected_recv=31545 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=34816 expected_recv=32406 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=34816 expected_recv=32124 expected_overflow=0 exact_recv_per_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=34816 expected_recv=32124 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=34816 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=34816 expected_recv=32136 expected_overflow=0 exact_recv_per_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=34816 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=34816 expected_recv=32733 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=34816 expected_recv=32136 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=34816 expected_recv=32766 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32384 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=34816 expected_recv=31475 expected_overflow=0 exact_recv_per_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=34816 expected_recv=31475 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=31872 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=34816 expected_recv=32639 expected_overflow=0 exact_recv_per_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=34816 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=34816 expected_recv=32639 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=34816 expected_recv=32872 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=34816 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=34816 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=34816 expected_recv=32773 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=34816 expected_recv=32755 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=34816 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=34816 expected_recv=32780 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=34816 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=34816 expected_recv=32174 expected_overflow=0 exact_recv_per_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=34816 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=34816 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=34816 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=34816 expected_recv=32174 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=34816 expected_recv=32706 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=32384 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=34816 expected_recv=32772 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=34816 expected_recv=32017 expected_overflow=0 exact_recv_per_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=34816 expected_recv=32017 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=34816 expected_recv=32514 expected_overflow=0 exact_recv_per_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=34816 expected_recv=32514 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=34816 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=34816 expected_recv=33052 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=34816 expected_recv=32789 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=34816 expected_recv=33052 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=34816 expected_recv=32331 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=34816 expected_recv=32331 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=34816 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=34816 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=34816 expected_recv=31938 expected_overflow=0 exact_recv_per_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=34816 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=34816 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=34816 expected_recv=31938 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=34816 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=34816 expected_recv=32762 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=34816 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=34816 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=34816 expected_recv=31466 expected_overflow=0 exact_recv_per_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=34816 expected_recv=32112 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=34816 expected_recv=32774 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=34816 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=34816 expected_recv=31466 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=34816 expected_recv=32112 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=31744 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=34816 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=34816 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=34816 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=34816 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=34816 expected_recv=32289 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=34816 expected_recv=32770 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=34816 expected_recv=32765 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=34816 expected_recv=32289 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32384 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=34816 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=34816 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=34816 expected_recv=32435 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=34816 expected_recv=32772 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=34816 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[41560, 48383, 38205, 24408, 16379, 27795, 26362, 39052] avg=32768.0000 max=48383 max_rank=1 min=16379 min_rank=4 imbalance_ratio=1.476532 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=34816 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=34816 expected_recv=32920 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[41560, 48383, 38205, 24408, 16379, 27795, 26362, 39052] avg=32768.0000 max=48383 max_rank=1 min=16379 min_rank=4 imbalance_ratio=1.476532 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=34816 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[41560, 48383, 38205, 24408, 16379, 27795, 26362, 39052] avg=32768.0000 max=48383 max_rank=1 min=16379 min_rank=4 imbalance_ratio=1.476532 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[41560, 48383, 38205, 24408, 16379, 27795, 26362, 39052] avg=32768.0000 max=48383 max_rank=1 min=16379 min_rank=4 imbalance_ratio=1.476532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15286 max_load_reduction_pct=0.315937 moved_to_replicas=34816 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15286 max_load_reduction_pct=0.315937 moved_to_replicas=34816 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[41560, 48383, 38205, 24408, 16379, 27795, 26362, 39052] avg=32768.0000 max=48383 max_rank=1 min=16379 min_rank=4 imbalance_ratio=1.476532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[41560, 48383, 38205, 24408, 16379, 27795, 26362, 39052] avg=32768.0000 max=48383 max_rank=1 min=16379 min_rank=4 imbalance_ratio=1.476532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15286 max_load_reduction_pct=0.315937 moved_to_replicas=34816 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15286 max_load_reduction_pct=0.315937 moved_to_replicas=34816 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[41560, 48383, 38205, 24408, 16379, 27795, 26362, 39052] avg=32768.0000 max=48383 max_rank=1 min=16379 min_rank=4 imbalance_ratio=1.476532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[65498, 18454, 49224, 32995, 4, 32814, 16, 63139] avg=32768.0000 max=65498 max_rank=0 min=4 min_rank=4 imbalance_ratio=1.998840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[41560, 48383, 38205, 24408, 16379, 27795, 26362, 39052] avg=32768.0000 max=48383 max_rank=1 min=16379 min_rank=4 imbalance_ratio=1.476532 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15286 max_load_reduction_pct=0.315937 moved_to_replicas=34816 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15286 max_load_reduction_pct=0.315937 moved_to_replicas=34816 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[65498, 18454, 49224, 32995, 4, 32814, 16, 63139] avg=32768.0000 max=65498 max_rank=0 min=4 min_rank=4 imbalance_ratio=1.998840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15286 max_load_reduction_pct=0.315937 moved_to_replicas=34816 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[65498, 18454, 49224, 32995, 4, 32814, 16, 63139] avg=32768.0000 max=65498 max_rank=0 min=4 min_rank=4 imbalance_ratio=1.998840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] avg=32768.0000 max=33096 max_rank=0 min=31545 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32402 max_load_reduction_pct=0.494702 moved_to_replicas=78573 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[65498, 18454, 49224, 32995, 4, 32814, 16, 63139] avg=32768.0000 max=65498 max_rank=0 min=4 min_rank=4 imbalance_ratio=1.998840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[33097, 33097, 33096, 33094, 30472, 33096, 33096, 33096] avg=32768.0000 max=33097 max_rank=0 min=30472 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=15286 max_load_reduction_pct=0.315937 moved_to_replicas=34816 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] avg=32768.0000 max=33096 max_rank=0 min=31545 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32402 max_load_reduction_pct=0.494702 moved_to_replicas=78573 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[65498, 18454, 49224, 32995, 4, 32814, 16, 63139] avg=32768.0000 max=65498 max_rank=0 min=4 min_rank=4 imbalance_ratio=1.998840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[65498, 18454, 49224, 32995, 4, 32814, 16, 63139] avg=32768.0000 max=65498 max_rank=0 min=4 min_rank=4 imbalance_ratio=1.998840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] avg=32768.0000 max=33096 max_rank=0 min=31545 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32402 max_load_reduction_pct=0.494702 moved_to_replicas=78573 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] avg=32768.0000 max=33096 max_rank=0 min=31545 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32402 max_load_reduction_pct=0.494702 moved_to_replicas=78573 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[65498, 18454, 49224, 32995, 4, 32814, 16, 63139] avg=32768.0000 max=65498 max_rank=0 min=4 min_rank=4 imbalance_ratio=1.998840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[97971, 32733, 26, 61959, 38821, 9606, 1, 21027] avg=32768.0000 max=97971 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] avg=32768.0000 max=33096 max_rank=0 min=31545 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32402 max_load_reduction_pct=0.494702 moved_to_replicas=78573 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[65498, 18454, 49224, 32995, 4, 32814, 16, 63139] avg=32768.0000 max=65498 max_rank=0 min=4 min_rank=4 imbalance_ratio=1.998840 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[97971, 32733, 26, 61959, 38821, 9606, 1, 21027] avg=32768.0000 max=97971 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] avg=32768.0000 max=33096 max_rank=0 min=31545 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32402 max_load_reduction_pct=0.494702 moved_to_replicas=78573 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] avg=32768.0000 max=33096 max_rank=0 min=31545 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32402 max_load_reduction_pct=0.494702 moved_to_replicas=78573 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] avg=32768.0000 max=33097 max_rank=4 min=32124 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64874 max_load_reduction_pct=0.662176 moved_to_replicas=99463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[97971, 32733, 26, 61959, 38821, 9606, 1, 21027] avg=32768.0000 max=97971 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[97971, 32733, 26, 61959, 38821, 9606, 1, 21027] avg=32768.0000 max=97971 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[33096, 33096, 33096, 32995, 32406, 32814, 31545, 33096] avg=32768.0000 max=33096 max_rank=0 min=31545 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=32402 max_load_reduction_pct=0.494702 moved_to_replicas=78573 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] avg=32768.0000 max=33097 max_rank=4 min=32124 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64874 max_load_reduction_pct=0.662176 moved_to_replicas=99463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[97971, 32733, 26, 61959, 38821, 9606, 1, 21027] avg=32768.0000 max=97971 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] avg=32768.0000 max=33097 max_rank=4 min=32124 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64874 max_load_reduction_pct=0.662176 moved_to_replicas=99463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[97971, 32733, 26, 61959, 38821, 9606, 1, 21027] avg=32768.0000 max=97971 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] avg=32768.0000 max=33097 max_rank=4 min=32124 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64874 max_load_reduction_pct=0.662176 moved_to_replicas=99463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[97971, 32733, 26, 61959, 38821, 9606, 1, 21027] avg=32768.0000 max=97971 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[46527, 32639, 33264, 39395, 32773, 11578, 186, 65782] avg=32768.0000 max=65782 max_rank=7 min=186 min_rank=6 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] avg=32768.0000 max=33097 max_rank=4 min=32124 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64874 max_load_reduction_pct=0.662176 moved_to_replicas=99463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[97971, 32733, 26, 61959, 38821, 9606, 1, 21027] avg=32768.0000 max=97971 max_rank=0 min=1 min_rank=6 imbalance_ratio=2.989838 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[46527, 32639, 33264, 39395, 32773, 11578, 186, 65782] avg=32768.0000 max=65782 max_rank=7 min=186 min_rank=6 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] avg=32768.0000 max=33097 max_rank=4 min=32124 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64874 max_load_reduction_pct=0.662176 moved_to_replicas=99463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] avg=32768.0000 max=33097 max_rank=0 min=31475 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32685 max_load_reduction_pct=0.496868 moved_to_replicas=52584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] avg=32768.0000 max=33097 max_rank=4 min=32124 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64874 max_load_reduction_pct=0.662176 moved_to_replicas=99463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[46527, 32639, 33264, 39395, 32773, 11578, 186, 65782] avg=32768.0000 max=65782 max_rank=7 min=186 min_rank=6 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[46527, 32639, 33264, 39395, 32773, 11578, 186, 65782] avg=32768.0000 max=65782 max_rank=7 min=186 min_rank=6 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] avg=32768.0000 max=33097 max_rank=0 min=31475 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32685 max_load_reduction_pct=0.496868 moved_to_replicas=52584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[33096, 32733, 32136, 33096, 33097, 33096, 32766, 32124] avg=32768.0000 max=33097 max_rank=4 min=32124 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=64874 max_load_reduction_pct=0.662176 moved_to_replicas=99463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[46527, 32639, 33264, 39395, 32773, 11578, 186, 65782] avg=32768.0000 max=65782 max_rank=7 min=186 min_rank=6 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] avg=32768.0000 max=33097 max_rank=0 min=31475 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32685 max_load_reduction_pct=0.496868 moved_to_replicas=52584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] avg=32768.0000 max=33097 max_rank=0 min=31475 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32685 max_load_reduction_pct=0.496868 moved_to_replicas=52584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[46527, 32639, 33264, 39395, 32773, 11578, 186, 65782] avg=32768.0000 max=65782 max_rank=7 min=186 min_rank=6 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 14, 65526, 32755, 44, 66094] avg=32768.0000 max=66094 max_rank=7 min=14 min_rank=3 imbalance_ratio=2.017029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[46527, 32639, 33264, 39395, 32773, 11578, 186, 65782] avg=32768.0000 max=65782 max_rank=7 min=186 min_rank=6 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 14, 65526, 32755, 44, 66094] avg=32768.0000 max=66094 max_rank=7 min=14 min_rank=3 imbalance_ratio=2.017029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[46527, 32639, 33264, 39395, 32773, 11578, 186, 65782] avg=32768.0000 max=65782 max_rank=7 min=186 min_rank=6 imbalance_ratio=2.007507 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] avg=32768.0000 max=33097 max_rank=0 min=31475 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32685 max_load_reduction_pct=0.496868 moved_to_replicas=52584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] avg=32768.0000 max=33097 max_rank=0 min=31475 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32685 max_load_reduction_pct=0.496868 moved_to_replicas=52584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] avg=32768.0000 max=33096 max_rank=4 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32998 max_load_reduction_pct=0.499259 moved_to_replicas=65428 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] avg=32768.0000 max=33097 max_rank=0 min=31475 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32685 max_load_reduction_pct=0.496868 moved_to_replicas=52584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 14, 65526, 32755, 44, 66094] avg=32768.0000 max=66094 max_rank=7 min=14 min_rank=3 imbalance_ratio=2.017029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 14, 65526, 32755, 44, 66094] avg=32768.0000 max=66094 max_rank=7 min=14 min_rank=3 imbalance_ratio=2.017029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] avg=32768.0000 max=33096 max_rank=4 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32998 max_load_reduction_pct=0.499259 moved_to_replicas=65428 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[33097, 32639, 33096, 33096, 32773, 31475, 32872, 33096] avg=32768.0000 max=33097 max_rank=0 min=31475 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=32685 max_load_reduction_pct=0.496868 moved_to_replicas=52584 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] avg=32768.0000 max=33096 max_rank=4 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32998 max_load_reduction_pct=0.499259 moved_to_replicas=65428 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 14, 65526, 32755, 44, 66094] avg=32768.0000 max=66094 max_rank=7 min=14 min_rank=3 imbalance_ratio=2.017029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[65536, 74, 32768, 9, 32789, 1, 96871, 34096] avg=32768.0000 max=96871 max_rank=6 min=1 min_rank=5 imbalance_ratio=2.956268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 14, 65526, 32755, 44, 66094] avg=32768.0000 max=66094 max_rank=7 min=14 min_rank=3 imbalance_ratio=2.017029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] avg=32768.0000 max=33096 max_rank=4 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32998 max_load_reduction_pct=0.499259 moved_to_replicas=65428 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 14, 65526, 32755, 44, 66094] avg=32768.0000 max=66094 max_rank=7 min=14 min_rank=3 imbalance_ratio=2.017029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[65536, 74, 32768, 9, 32789, 1, 96871, 34096] avg=32768.0000 max=96871 max_rank=6 min=1 min_rank=5 imbalance_ratio=2.956268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 14, 65526, 32755, 44, 66094] avg=32768.0000 max=66094 max_rank=7 min=14 min_rank=3 imbalance_ratio=2.017029 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] avg=32768.0000 max=33096 max_rank=4 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32998 max_load_reduction_pct=0.499259 moved_to_replicas=65428 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32017 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63775 max_load_reduction_pct=0.658350 moved_to_replicas=97215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] avg=32768.0000 max=33096 max_rank=4 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32998 max_load_reduction_pct=0.499259 moved_to_replicas=65428 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] avg=32768.0000 max=33096 max_rank=4 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32998 max_load_reduction_pct=0.499259 moved_to_replicas=65428 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[65536, 74, 32768, 9, 32789, 1, 96871, 34096] avg=32768.0000 max=96871 max_rank=6 min=1 min_rank=5 imbalance_ratio=2.956268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32017 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63775 max_load_reduction_pct=0.658350 moved_to_replicas=97215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[65536, 74, 32768, 9, 32789, 1, 96871, 34096] avg=32768.0000 max=96871 max_rank=6 min=1 min_rank=5 imbalance_ratio=2.956268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32772, 32765, 32174, 32780, 33096, 32755, 32706, 33096] avg=32768.0000 max=33096 max_rank=4 min=32174 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=32998 max_load_reduction_pct=0.499259 moved_to_replicas=65428 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32017 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63775 max_load_reduction_pct=0.658350 moved_to_replicas=97215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[65536, 74, 32768, 9, 32789, 1, 96871, 34096] avg=32768.0000 max=96871 max_rank=6 min=1 min_rank=5 imbalance_ratio=2.956268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32017 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63775 max_load_reduction_pct=0.658350 moved_to_replicas=97215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[0, 65413, 0, 61624, 6, 4525, 130562, 14] avg=32768.0000 max=130562 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.984436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[65536, 74, 32768, 9, 32789, 1, 96871, 34096] avg=32768.0000 max=96871 max_rank=6 min=1 min_rank=5 imbalance_ratio=2.956268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[65536, 74, 32768, 9, 32789, 1, 96871, 34096] avg=32768.0000 max=96871 max_rank=6 min=1 min_rank=5 imbalance_ratio=2.956268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[0, 65413, 0, 61624, 6, 4525, 130562, 14] avg=32768.0000 max=130562 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.984436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[65536, 74, 32768, 9, 32789, 1, 96871, 34096] avg=32768.0000 max=96871 max_rank=6 min=1 min_rank=5 imbalance_ratio=2.956268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32017 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63775 max_load_reduction_pct=0.658350 moved_to_replicas=97215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] avg=32768.0000 max=33097 max_rank=3 min=31938 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=97465 max_load_reduction_pct=0.746504 moved_to_replicas=158311 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32017 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63775 max_load_reduction_pct=0.658350 moved_to_replicas=97215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32017 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63775 max_load_reduction_pct=0.658350 moved_to_replicas=97215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[0, 65413, 0, 61624, 6, 4525, 130562, 14] avg=32768.0000 max=130562 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.984436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] avg=32768.0000 max=33097 max_rank=3 min=31938 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=97465 max_load_reduction_pct=0.746504 moved_to_replicas=158311 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[0, 65413, 0, 61624, 6, 4525, 130562, 14] avg=32768.0000 max=130562 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.984436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[33096, 32514, 32768, 32017, 32789, 32768, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=32017 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=63775 max_load_reduction_pct=0.658350 moved_to_replicas=97215 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] avg=32768.0000 max=33097 max_rank=3 min=31938 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=97465 max_load_reduction_pct=0.746504 moved_to_replicas=158311 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32765, 5071, 65534, 49475, 1, 32768, 0, 76530] avg=32768.0000 max=76530 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] avg=32768.0000 max=33097 max_rank=3 min=31938 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=97465 max_load_reduction_pct=0.746504 moved_to_replicas=158311 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[0, 65413, 0, 61624, 6, 4525, 130562, 14] avg=32768.0000 max=130562 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.984436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[0, 65413, 0, 61624, 6, 4525, 130562, 14] avg=32768.0000 max=130562 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.984436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[0, 65413, 0, 61624, 6, 4525, 130562, 14] avg=32768.0000 max=130562 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.984436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32765, 5071, 65534, 49475, 1, 32768, 0, 76530] avg=32768.0000 max=76530 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[0, 65413, 0, 61624, 6, 4525, 130562, 14] avg=32768.0000 max=130562 max_rank=6 min=0 min_rank=0 imbalance_ratio=3.984436 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] avg=32768.0000 max=33097 max_rank=3 min=31466 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=43433 max_load_reduction_pct=0.567529 moved_to_replicas=92251 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32765, 5071, 65534, 49475, 1, 32768, 0, 76530] avg=32768.0000 max=76530 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] avg=32768.0000 max=33097 max_rank=3 min=31938 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=97465 max_load_reduction_pct=0.746504 moved_to_replicas=158311 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] avg=32768.0000 max=33097 max_rank=3 min=31938 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=97465 max_load_reduction_pct=0.746504 moved_to_replicas=158311 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] avg=32768.0000 max=33097 max_rank=3 min=31938 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=97465 max_load_reduction_pct=0.746504 moved_to_replicas=158311 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] avg=32768.0000 max=33097 max_rank=3 min=31466 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=43433 max_load_reduction_pct=0.567529 moved_to_replicas=92251 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[32767, 33096, 32767, 33097, 31938, 33052, 33096, 32331] avg=32768.0000 max=33097 max_rank=3 min=31938 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=97465 max_load_reduction_pct=0.746504 moved_to_replicas=158311 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32765, 5071, 65534, 49475, 1, 32768, 0, 76530] avg=32768.0000 max=76530 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] avg=32768.0000 max=33097 max_rank=3 min=31466 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=43433 max_load_reduction_pct=0.567529 moved_to_replicas=92251 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 26709, 32765, 32774, 77364, 26996] avg=32768.0000 max=77364 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.360962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] avg=32768.0000 max=33097 max_rank=3 min=31466 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=43433 max_load_reduction_pct=0.567529 moved_to_replicas=92251 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32765, 5071, 65534, 49475, 1, 32768, 0, 76530] avg=32768.0000 max=76530 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32765, 5071, 65534, 49475, 1, 32768, 0, 76530] avg=32768.0000 max=76530 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 26709, 32765, 32774, 77364, 26996] avg=32768.0000 max=77364 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.360962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32765, 5071, 65534, 49475, 1, 32768, 0, 76530] avg=32768.0000 max=76530 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32765, 5071, 65534, 49475, 1, 32768, 0, 76530] avg=32768.0000 max=76530 max_rank=7 min=0 min_rank=6 imbalance_ratio=2.335510 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] avg=32768.0000 max=33095 max_rank=3 min=32112 min_rank=7 imbalance_ratio=1.009979 max_load_reduction=44269 max_load_reduction_pct=0.572217 moved_to_replicas=44268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 26709, 32765, 32774, 77364, 26996] avg=32768.0000 max=77364 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.360962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] avg=32768.0000 max=33097 max_rank=3 min=31466 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=43433 max_load_reduction_pct=0.567529 moved_to_replicas=92251 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] avg=32768.0000 max=33097 max_rank=3 min=31466 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=43433 max_load_reduction_pct=0.567529 moved_to_replicas=92251 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] avg=32768.0000 max=33095 max_rank=3 min=32112 min_rank=7 imbalance_ratio=1.009979 max_load_reduction=44269 max_load_reduction_pct=0.572217 moved_to_replicas=44268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] avg=32768.0000 max=33097 max_rank=3 min=31466 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=43433 max_load_reduction_pct=0.567529 moved_to_replicas=92251 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32765, 33094, 33096, 33097, 31466, 32768, 32762, 33096] avg=32768.0000 max=33097 max_rank=3 min=31466 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=43433 max_load_reduction_pct=0.567529 moved_to_replicas=92251 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 26709, 32765, 32774, 77364, 26996] avg=32768.0000 max=77364 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.360962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32920, 65385, 0, 65531, 32766, 32770, 32772, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] avg=32768.0000 max=33095 max_rank=3 min=32112 min_rank=7 imbalance_ratio=1.009979 max_load_reduction=44269 max_load_reduction_pct=0.572217 moved_to_replicas=44268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] avg=32768.0000 max=33095 max_rank=3 min=32112 min_rank=7 imbalance_ratio=1.009979 max_load_reduction=44269 max_load_reduction_pct=0.572217 moved_to_replicas=44268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 26709, 32765, 32774, 77364, 26996] avg=32768.0000 max=77364 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.360962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 26709, 32765, 32774, 77364, 26996] avg=32768.0000 max=77364 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.360962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32920, 65385, 0, 65531, 32766, 32770, 32772, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 26709, 32765, 32774, 77364, 26996] avg=32768.0000 max=77364 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.360962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64724 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32768, 0, 32768, 26709, 32765, 32774, 77364, 26996] avg=32768.0000 max=77364 max_rank=6 min=0 min_rank=1 imbalance_ratio=2.360962 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32920, 65385, 0, 65531, 32766, 32770, 32772, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] avg=32768.0000 max=33095 max_rank=3 min=32112 min_rank=7 imbalance_ratio=1.009979 max_load_reduction=44269 max_load_reduction_pct=0.572217 moved_to_replicas=44268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] avg=32768.0000 max=33095 max_rank=3 min=32112 min_rank=7 imbalance_ratio=1.009979 max_load_reduction=44269 max_load_reduction_pct=0.572217 moved_to_replicas=44268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64724 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] avg=32768.0000 max=33095 max_rank=3 min=32112 min_rank=7 imbalance_ratio=1.009979 max_load_reduction=44269 max_load_reduction_pct=0.572217 moved_to_replicas=44268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32920, 65385, 0, 65531, 32766, 32770, 32772, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32768, 32767, 32768, 33095, 32765, 32774, 33095, 32112] avg=32768.0000 max=33095 max_rank=3 min=32112 min_rank=7 imbalance_ratio=1.009979 max_load_reduction=44269 max_load_reduction_pct=0.572217 moved_to_replicas=44268 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64724 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64724 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32920, 65385, 0, 65531, 32766, 32770, 32772, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32920, 65385, 0, 65531, 32766, 32770, 32772, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32920, 65385, 0, 65531, 32766, 32770, 32772, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32920, 65385, 0, 65531, 32766, 32770, 32772, 0] avg=32768.0000 max=65531 max_rank=3 min=0 min_rank=2 imbalance_ratio=1.999847 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64724 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64724 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64724 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32920, 33096, 32435, 33096, 32766, 32770, 32772, 32289] avg=32768.0000 max=33096 max_rank=1 min=32289 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32435 max_load_reduction_pct=0.494957 moved_to_replicas=64724 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=10 total_expected_recv=327368 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=18048 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=10 total_expected_recv=327432 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=18304 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=10 total_expected_recv=329463 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16256 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=10 total_expected_recv=328897 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16896 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=10 total_expected_recv=327806 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=17024 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=10 total_expected_recv=327432 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=18816 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=10 total_expected_recv=329474 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16512 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=10 total_expected_recv=323568 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=22016 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 18:02:39] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 233.5 | throughput per GPU (TFLOP/s/GPU): 528.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.973323E+00 | loss scale: 1.0 | grad norm: 4.563 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=34816 expected_recv=31677 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=34816 expected_recv=31890 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=34816 expected_recv=31677 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=34816 expected_recv=31890 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=34816 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=34816 expected_recv=31633 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=34816 expected_recv=31936 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=34816 expected_recv=33098 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=34816 expected_recv=31633 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=34816 expected_recv=31936 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=34816 expected_recv=32000 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=34816 expected_recv=32000 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=34816 expected_recv=31900 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=34816 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=34816 expected_recv=31900 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=34816 expected_recv=32764 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=34816 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=34816 expected_recv=33098 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=34816 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=34816 expected_recv=32257 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=34816 expected_recv=32807 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=34816 expected_recv=31600 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=34816 expected_recv=32257 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=34816 expected_recv=31600 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=32384 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=31744 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=34816 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=34816 expected_recv=31070 expected_overflow=0 exact_recv_per_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=34816 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=34816 expected_recv=32776 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=34816 expected_recv=31070 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=34816 expected_recv=32817 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=31616 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32943 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32710 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32660 expected_overflow=0 exact_recv_per_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32759 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=34816 expected_recv=32714 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32660 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=34816 expected_recv=32714 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=34816 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=34816 expected_recv=32770 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=34816 expected_recv=31180 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=34816 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=34816 expected_recv=31180 actual_valid_recv=31232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=34816 expected_recv=32778 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=31232 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=34816 expected_recv=33039 expected_overflow=0 exact_recv_per_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=34816 expected_recv=31848 expected_overflow=0 exact_recv_per_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=34816 expected_recv=33039 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=34816 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=34816 expected_recv=31848 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=34816 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=34816 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=34816 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=34816 expected_recv=32744 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=34816 expected_recv=32775 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=34816 expected_recv=31510 expected_overflow=0 exact_recv_per_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=34816 expected_recv=31510 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=31616 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=34816 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=34816 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=34816 expected_recv=32212 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=34816 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=34816 expected_recv=32381 expected_overflow=0 exact_recv_per_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=34816 expected_recv=32771 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=34816 expected_recv=32212 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=34816 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=34816 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=34816 expected_recv=32381 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=34816 expected_recv=32770 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=34816 expected_recv=32759 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=34816 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[40824, 42422, 34310, 27298, 28534, 26263, 26504, 35989] avg=32768.0000 max=42422 max_rank=1 min=26263 min_rank=5 imbalance_ratio=1.294617 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=34816 expected_recv=32760 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=34816 expected_recv=32641 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=34816 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[40824, 42422, 34310, 27298, 28534, 26263, 26504, 35989] avg=32768.0000 max=42422 max_rank=1 min=26263 min_rank=5 imbalance_ratio=1.294617 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=34816 expected_recv=32641 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[40824, 42422, 34310, 27298, 28534, 26263, 26504, 35989] avg=32768.0000 max=42422 max_rank=1 min=26263 min_rank=5 imbalance_ratio=1.294617 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=34816 expected_recv=32810 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[40824, 42422, 34310, 27298, 28534, 26263, 26504, 35989] avg=32768.0000 max=42422 max_rank=1 min=26263 min_rank=5 imbalance_ratio=1.294617 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] avg=32768.0000 max=33097 max_rank=2 min=31677 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9325 max_load_reduction_pct=0.219815 moved_to_replicas=21161 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] avg=32768.0000 max=33097 max_rank=2 min=31677 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9325 max_load_reduction_pct=0.219815 moved_to_replicas=21161 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] avg=32768.0000 max=33097 max_rank=2 min=31677 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9325 max_load_reduction_pct=0.219815 moved_to_replicas=21161 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[40824, 42422, 34310, 27298, 28534, 26263, 26504, 35989] avg=32768.0000 max=42422 max_rank=1 min=26263 min_rank=5 imbalance_ratio=1.294617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] avg=32768.0000 max=33097 max_rank=2 min=31677 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9325 max_load_reduction_pct=0.219815 moved_to_replicas=21161 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[40824, 42422, 34310, 27298, 28534, 26263, 26504, 35989] avg=32768.0000 max=42422 max_rank=1 min=26263 min_rank=5 imbalance_ratio=1.294617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[40824, 42422, 34310, 27298, 28534, 26263, 26504, 35989] avg=32768.0000 max=42422 max_rank=1 min=26263 min_rank=5 imbalance_ratio=1.294617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[40824, 42422, 34310, 27298, 28534, 26263, 26504, 35989] avg=32768.0000 max=42422 max_rank=1 min=26263 min_rank=5 imbalance_ratio=1.294617 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] avg=32768.0000 max=33097 max_rank=2 min=31677 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9325 max_load_reduction_pct=0.219815 moved_to_replicas=21161 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[59208, 59739, 36821, 66200, 3588, 1174, 916, 34498] avg=32768.0000 max=66200 max_rank=3 min=916 min_rank=6 imbalance_ratio=2.020264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[59208, 59739, 36821, 66200, 3588, 1174, 916, 34498] avg=32768.0000 max=66200 max_rank=3 min=916 min_rank=6 imbalance_ratio=2.020264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] avg=32768.0000 max=33097 max_rank=2 min=31677 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9325 max_load_reduction_pct=0.219815 moved_to_replicas=21161 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[59208, 59739, 36821, 66200, 3588, 1174, 916, 34498] avg=32768.0000 max=66200 max_rank=3 min=916 min_rank=6 imbalance_ratio=2.020264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[59208, 59739, 36821, 66200, 3588, 1174, 916, 34498] avg=32768.0000 max=66200 max_rank=3 min=916 min_rank=6 imbalance_ratio=2.020264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] avg=32768.0000 max=33097 max_rank=2 min=31677 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9325 max_load_reduction_pct=0.219815 moved_to_replicas=21161 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[33096, 33096, 33097, 33096, 31677, 33096, 31890, 33096] avg=32768.0000 max=33097 max_rank=2 min=31677 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=9325 max_load_reduction_pct=0.219815 moved_to_replicas=21161 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] avg=32768.0000 max=33098 max_rank=5 min=31633 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=33102 max_load_reduction_pct=0.500030 moved_to_replicas=90986 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] avg=32768.0000 max=33098 max_rank=5 min=31633 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=33102 max_load_reduction_pct=0.500030 moved_to_replicas=90986 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[59208, 59739, 36821, 66200, 3588, 1174, 916, 34498] avg=32768.0000 max=66200 max_rank=3 min=916 min_rank=6 imbalance_ratio=2.020264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] avg=32768.0000 max=33098 max_rank=5 min=31633 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=33102 max_load_reduction_pct=0.500030 moved_to_replicas=90986 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] avg=32768.0000 max=33098 max_rank=5 min=31633 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=33102 max_load_reduction_pct=0.500030 moved_to_replicas=90986 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[59208, 59739, 36821, 66200, 3588, 1174, 916, 34498] avg=32768.0000 max=66200 max_rank=3 min=916 min_rank=6 imbalance_ratio=2.020264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[59208, 59739, 36821, 66200, 3588, 1174, 916, 34498] avg=32768.0000 max=66200 max_rank=3 min=916 min_rank=6 imbalance_ratio=2.020264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[59208, 59739, 36821, 66200, 3588, 1174, 916, 34498] avg=32768.0000 max=66200 max_rank=3 min=916 min_rank=6 imbalance_ratio=2.020264 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[39479, 37497, 36342, 83466, 33108, 31888, 2, 362] avg=32768.0000 max=83466 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.547180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] avg=32768.0000 max=33098 max_rank=5 min=31633 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=33102 max_load_reduction_pct=0.500030 moved_to_replicas=90986 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[39479, 37497, 36342, 83466, 33108, 31888, 2, 362] avg=32768.0000 max=83466 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.547180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] avg=32768.0000 max=33098 max_rank=5 min=31633 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=33102 max_load_reduction_pct=0.500030 moved_to_replicas=90986 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[39479, 37497, 36342, 83466, 33108, 31888, 2, 362] avg=32768.0000 max=83466 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.547180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[39479, 37497, 36342, 83466, 33108, 31888, 2, 362] avg=32768.0000 max=83466 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.547180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] avg=32768.0000 max=33098 max_rank=5 min=31633 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=33102 max_load_reduction_pct=0.500030 moved_to_replicas=90986 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[33095, 33096, 33095, 33095, 31633, 33098, 31936, 33096] avg=32768.0000 max=33098 max_rank=5 min=31633 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=33102 max_load_reduction_pct=0.500030 moved_to_replicas=90986 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] avg=32768.0000 max=33097 max_rank=3 min=31900 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=50369 max_load_reduction_pct=0.603467 moved_to_replicas=64412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] avg=32768.0000 max=33097 max_rank=3 min=31900 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=50369 max_load_reduction_pct=0.603467 moved_to_replicas=64412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] avg=32768.0000 max=33097 max_rank=3 min=31900 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=50369 max_load_reduction_pct=0.603467 moved_to_replicas=64412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] avg=32768.0000 max=33097 max_rank=3 min=31900 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=50369 max_load_reduction_pct=0.603467 moved_to_replicas=64412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[39479, 37497, 36342, 83466, 33108, 31888, 2, 362] avg=32768.0000 max=83466 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.547180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[39479, 37497, 36342, 83466, 33108, 31888, 2, 362] avg=32768.0000 max=83466 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.547180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[39479, 37497, 36342, 83466, 33108, 31888, 2, 362] avg=32768.0000 max=83466 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.547180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[43344, 4759, 31319, 10261, 32257, 55, 12696, 127453] avg=32768.0000 max=127453 max_rank=7 min=55 min_rank=5 imbalance_ratio=3.889557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[39479, 37497, 36342, 83466, 33108, 31888, 2, 362] avg=32768.0000 max=83466 max_rank=3 min=2 min_rank=6 imbalance_ratio=2.547180 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[43344, 4759, 31319, 10261, 32257, 55, 12696, 127453] avg=32768.0000 max=127453 max_rank=7 min=55 min_rank=5 imbalance_ratio=3.889557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] avg=32768.0000 max=33097 max_rank=3 min=31900 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=50369 max_load_reduction_pct=0.603467 moved_to_replicas=64412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[43344, 4759, 31319, 10261, 32257, 55, 12696, 127453] avg=32768.0000 max=127453 max_rank=7 min=55 min_rank=5 imbalance_ratio=3.889557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[43344, 4759, 31319, 10261, 32257, 55, 12696, 127453] avg=32768.0000 max=127453 max_rank=7 min=55 min_rank=5 imbalance_ratio=3.889557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] avg=32768.0000 max=33097 max_rank=3 min=31900 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=50369 max_load_reduction_pct=0.603467 moved_to_replicas=64412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] avg=32768.0000 max=33097 max_rank=3 min=31900 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=50369 max_load_reduction_pct=0.603467 moved_to_replicas=64412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31600 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=94355 max_load_reduction_pct=0.740312 moved_to_replicas=104605 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[33096, 33095, 33096, 33097, 33096, 31900, 32764, 32000] avg=32768.0000 max=33097 max_rank=3 min=31900 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=50369 max_load_reduction_pct=0.603467 moved_to_replicas=64412 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31600 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=94355 max_load_reduction_pct=0.740312 moved_to_replicas=104605 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31600 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=94355 max_load_reduction_pct=0.740312 moved_to_replicas=104605 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31600 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=94355 max_load_reduction_pct=0.740312 moved_to_replicas=104605 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[43344, 4759, 31319, 10261, 32257, 55, 12696, 127453] avg=32768.0000 max=127453 max_rank=7 min=55 min_rank=5 imbalance_ratio=3.889557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[43344, 4759, 31319, 10261, 32257, 55, 12696, 127453] avg=32768.0000 max=127453 max_rank=7 min=55 min_rank=5 imbalance_ratio=3.889557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[43344, 4759, 31319, 10261, 32257, 55, 12696, 127453] avg=32768.0000 max=127453 max_rank=7 min=55 min_rank=5 imbalance_ratio=3.889557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[13927, 4788, 9, 32817, 98205, 42484, 33099, 36815] avg=32768.0000 max=98205 max_rank=4 min=9 min_rank=2 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[13927, 4788, 9, 32817, 98205, 42484, 33099, 36815] avg=32768.0000 max=98205 max_rank=4 min=9 min_rank=2 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[43344, 4759, 31319, 10261, 32257, 55, 12696, 127453] avg=32768.0000 max=127453 max_rank=7 min=55 min_rank=5 imbalance_ratio=3.889557 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31600 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=94355 max_load_reduction_pct=0.740312 moved_to_replicas=104605 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[13927, 4788, 9, 32817, 98205, 42484, 33099, 36815] avg=32768.0000 max=98205 max_rank=4 min=9 min_rank=2 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[13927, 4788, 9, 32817, 98205, 42484, 33099, 36815] avg=32768.0000 max=98205 max_rank=4 min=9 min_rank=2 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31600 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=94355 max_load_reduction_pct=0.740312 moved_to_replicas=104605 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31070 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65108 max_load_reduction_pct=0.662980 moved_to_replicas=78219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31600 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=94355 max_load_reduction_pct=0.740312 moved_to_replicas=104605 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31070 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65108 max_load_reduction_pct=0.662980 moved_to_replicas=78219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[33096, 33094, 31600, 33096, 32257, 32807, 33096, 33098] avg=32768.0000 max=33098 max_rank=7 min=31600 min_rank=2 imbalance_ratio=1.010071 max_load_reduction=94355 max_load_reduction_pct=0.740312 moved_to_replicas=104605 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31070 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65108 max_load_reduction_pct=0.662980 moved_to_replicas=78219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31070 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65108 max_load_reduction_pct=0.662980 moved_to_replicas=78219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[13927, 4788, 9, 32817, 98205, 42484, 33099, 36815] avg=32768.0000 max=98205 max_rank=4 min=9 min_rank=2 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[13927, 4788, 9, 32817, 98205, 42484, 33099, 36815] avg=32768.0000 max=98205 max_rank=4 min=9 min_rank=2 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[13927, 4788, 9, 32817, 98205, 42484, 33099, 36815] avg=32768.0000 max=98205 max_rank=4 min=9 min_rank=2 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31070 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65108 max_load_reduction_pct=0.662980 moved_to_replicas=78219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[13927, 4788, 9, 32817, 98205, 42484, 33099, 36815] avg=32768.0000 max=98205 max_rank=4 min=9 min_rank=2 imbalance_ratio=2.996979 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31070 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65108 max_load_reduction_pct=0.662980 moved_to_replicas=78219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31070 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65108 max_load_reduction_pct=0.662980 moved_to_replicas=78219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[31070, 33096, 32776, 32817, 33096, 33097, 33096, 33096] avg=32768.0000 max=33097 max_rank=5 min=31070 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=65108 max_load_reduction_pct=0.662980 moved_to_replicas=78219 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[2154, 10843, 16504, 34081, 3, 65222, 100623, 32714] avg=32768.0000 max=100623 max_rank=6 min=3 min_rank=4 imbalance_ratio=3.070770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[2154, 10843, 16504, 34081, 3, 65222, 100623, 32714] avg=32768.0000 max=100623 max_rank=6 min=3 min_rank=4 imbalance_ratio=3.070770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[2154, 10843, 16504, 34081, 3, 65222, 100623, 32714] avg=32768.0000 max=100623 max_rank=6 min=3 min_rank=4 imbalance_ratio=3.070770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[2154, 10843, 16504, 34081, 3, 65222, 100623, 32714] avg=32768.0000 max=100623 max_rank=6 min=3 min_rank=4 imbalance_ratio=3.070770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] avg=32768.0000 max=33096 max_rank=0 min=31180 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=67527 max_load_reduction_pct=0.671089 moved_to_replicas=100638 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] avg=32768.0000 max=33096 max_rank=0 min=31180 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=67527 max_load_reduction_pct=0.671089 moved_to_replicas=100638 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[32660, 32759, 32768, 32768, 32768, 32710, 32768, 32943] avg=32768.0000 max=32943 max_rank=7 min=32660 min_rank=0 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] avg=32768.0000 max=33096 max_rank=0 min=31180 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=67527 max_load_reduction_pct=0.671089 moved_to_replicas=100638 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] avg=32768.0000 max=33096 max_rank=0 min=31180 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=67527 max_load_reduction_pct=0.671089 moved_to_replicas=100638 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[2154, 10843, 16504, 34081, 3, 65222, 100623, 32714] avg=32768.0000 max=100623 max_rank=6 min=3 min_rank=4 imbalance_ratio=3.070770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32744, 44168, 32775, 86618, 22, 33039, 0, 32778] avg=32768.0000 max=86618 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.643372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[2154, 10843, 16504, 34081, 3, 65222, 100623, 32714] avg=32768.0000 max=100623 max_rank=6 min=3 min_rank=4 imbalance_ratio=3.070770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32744, 44168, 32775, 86618, 22, 33039, 0, 32778] avg=32768.0000 max=86618 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.643372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[2154, 10843, 16504, 34081, 3, 65222, 100623, 32714] avg=32768.0000 max=100623 max_rank=6 min=3 min_rank=4 imbalance_ratio=3.070770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[2154, 10843, 16504, 34081, 3, 65222, 100623, 32714] avg=32768.0000 max=100623 max_rank=6 min=3 min_rank=4 imbalance_ratio=3.070770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] avg=32768.0000 max=33096 max_rank=0 min=31180 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=67527 max_load_reduction_pct=0.671089 moved_to_replicas=100638 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] avg=32768.0000 max=33097 max_rank=1 min=31848 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=53521 max_load_reduction_pct=0.617897 moved_to_replicas=64594 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32744, 44168, 32775, 86618, 22, 33039, 0, 32778] avg=32768.0000 max=86618 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.643372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] avg=32768.0000 max=33096 max_rank=0 min=31180 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=67527 max_load_reduction_pct=0.671089 moved_to_replicas=100638 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] avg=32768.0000 max=33097 max_rank=1 min=31848 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=53521 max_load_reduction_pct=0.617897 moved_to_replicas=64594 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32744, 44168, 32775, 86618, 22, 33039, 0, 32778] avg=32768.0000 max=86618 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.643372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] avg=32768.0000 max=33096 max_rank=0 min=31180 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=67527 max_load_reduction_pct=0.671089 moved_to_replicas=100638 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[33096, 33096, 31180, 33096, 32770, 33096, 33096, 32714] avg=32768.0000 max=33096 max_rank=0 min=31180 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=67527 max_load_reduction_pct=0.671089 moved_to_replicas=100638 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] avg=32768.0000 max=33097 max_rank=1 min=31848 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=53521 max_load_reduction_pct=0.617897 moved_to_replicas=64594 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[49860, 66, 32771, 3745, 0, 34166, 65894, 75642] avg=32768.0000 max=75642 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.308411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32744, 44168, 32775, 86618, 22, 33039, 0, 32778] avg=32768.0000 max=86618 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.643372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] avg=32768.0000 max=33097 max_rank=1 min=31848 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=53521 max_load_reduction_pct=0.617897 moved_to_replicas=64594 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[49860, 66, 32771, 3745, 0, 34166, 65894, 75642] avg=32768.0000 max=75642 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.308411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32744, 44168, 32775, 86618, 22, 33039, 0, 32778] avg=32768.0000 max=86618 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.643372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32744, 44168, 32775, 86618, 22, 33039, 0, 32778] avg=32768.0000 max=86618 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.643372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32744, 44168, 32775, 86618, 22, 33039, 0, 32778] avg=32768.0000 max=86618 max_rank=3 min=0 min_rank=6 imbalance_ratio=2.643372 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=31510 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=42544 max_load_reduction_pct=0.562439 moved_to_replicas=93178 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[49860, 66, 32771, 3745, 0, 34166, 65894, 75642] avg=32768.0000 max=75642 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.308411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] avg=32768.0000 max=33097 max_rank=1 min=31848 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=53521 max_load_reduction_pct=0.617897 moved_to_replicas=64594 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=31510 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=42544 max_load_reduction_pct=0.562439 moved_to_replicas=93178 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] avg=32768.0000 max=33097 max_rank=1 min=31848 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=53521 max_load_reduction_pct=0.617897 moved_to_replicas=64594 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[49860, 66, 32771, 3745, 0, 34166, 65894, 75642] avg=32768.0000 max=75642 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.308411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] avg=32768.0000 max=33097 max_rank=1 min=31848 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=53521 max_load_reduction_pct=0.617897 moved_to_replicas=64594 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32744, 33097, 32775, 33096, 31848, 33039, 32767, 32778] avg=32768.0000 max=33097 max_rank=1 min=31848 min_rank=4 imbalance_ratio=1.010040 max_load_reduction=53521 max_load_reduction_pct=0.617897 moved_to_replicas=64594 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=31510 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=42544 max_load_reduction_pct=0.562439 moved_to_replicas=93178 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[33222, 32641, 32760, 32810, 32759, 32770, 65182, 0] avg=32768.0000 max=65182 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.989197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[49860, 66, 32771, 3745, 0, 34166, 65894, 75642] avg=32768.0000 max=75642 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.308411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=31510 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=42544 max_load_reduction_pct=0.562439 moved_to_replicas=93178 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[33222, 32641, 32760, 32810, 32759, 32770, 65182, 0] avg=32768.0000 max=65182 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.989197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[49860, 66, 32771, 3745, 0, 34166, 65894, 75642] avg=32768.0000 max=75642 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.308411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[49860, 66, 32771, 3745, 0, 34166, 65894, 75642] avg=32768.0000 max=75642 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.308411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32086 max_load_reduction_pct=0.492252 moved_to_replicas=32212 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[49860, 66, 32771, 3745, 0, 34166, 65894, 75642] avg=32768.0000 max=75642 max_rank=7 min=0 min_rank=4 imbalance_ratio=2.308411 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[33222, 32641, 32760, 32810, 32759, 32770, 65182, 0] avg=32768.0000 max=65182 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.989197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=31510 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=42544 max_load_reduction_pct=0.562439 moved_to_replicas=93178 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32086 max_load_reduction_pct=0.492252 moved_to_replicas=32212 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=31510 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=42544 max_load_reduction_pct=0.562439 moved_to_replicas=93178 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=31510 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=42544 max_load_reduction_pct=0.562439 moved_to_replicas=93178 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[33222, 32641, 32760, 32810, 32759, 32770, 65182, 0] avg=32768.0000 max=65182 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.989197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32086 max_load_reduction_pct=0.492252 moved_to_replicas=32212 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[33098, 32381, 32771, 33096, 31510, 33096, 33096, 33096] avg=32768.0000 max=33098 max_rank=0 min=31510 min_rank=4 imbalance_ratio=1.010071 max_load_reduction=42544 max_load_reduction_pct=0.562439 moved_to_replicas=93178 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[33222, 32641, 32760, 32810, 32759, 32770, 65182, 0] avg=32768.0000 max=65182 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.989197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32086 max_load_reduction_pct=0.492252 moved_to_replicas=32212 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[33222, 32641, 32760, 32810, 32759, 32770, 65182, 0] avg=32768.0000 max=65182 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.989197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[33222, 32641, 32760, 32810, 32759, 32770, 65182, 0] avg=32768.0000 max=65182 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.989197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32086 max_load_reduction_pct=0.492252 moved_to_replicas=32212 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[33222, 32641, 32760, 32810, 32759, 32770, 65182, 0] avg=32768.0000 max=65182 max_rank=6 min=0 min_rank=7 imbalance_ratio=1.989197 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32086 max_load_reduction_pct=0.492252 moved_to_replicas=32212 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32086 max_load_reduction_pct=0.492252 moved_to_replicas=32212 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[33096, 32641, 32760, 32810, 32759, 32770, 33096, 32212] avg=32768.0000 max=33096 max_rank=0 min=32212 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=32086 max_load_reduction_pct=0.492252 moved_to_replicas=32212 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=10 total_expected_recv=328129 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16768 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=10 total_expected_recv=328709 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16256 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=10 total_expected_recv=327605 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=17920 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=10 total_expected_recv=323414 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=21888 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=10 total_expected_recv=328147 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16896 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=10 total_expected_recv=325918 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=20480 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=10 total_expected_recv=329451 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16640 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=10 total_expected_recv=330067 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=15104 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 18:02:39] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 239.5 | throughput per GPU (TFLOP/s/GPU): 515.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.758467E+00 | loss scale: 1.0 | grad norm: 4.311 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=34816 expected_recv=31907 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=34816 expected_recv=32550 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=34816 expected_recv=32205 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=34816 expected_recv=31907 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=34816 expected_recv=32550 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=34816 expected_recv=32205 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=34816 expected_recv=31050 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=34816 expected_recv=32517 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=34816 expected_recv=31050 actual_valid_recv=31872 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=34816 expected_recv=32517 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=31872 slack_slots=2944 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=34816 expected_recv=31705 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=34816 expected_recv=32516 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=34816 expected_recv=31705 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=34816 expected_recv=32516 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=34816 expected_recv=32443 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=34816 expected_recv=32443 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=34816 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=34816 expected_recv=32900 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=34816 expected_recv=31860 expected_overflow=0 exact_recv_per_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=34816 expected_recv=32559 expected_overflow=0 exact_recv_per_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=34816 expected_recv=32442 expected_overflow=0 exact_recv_per_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=34816 expected_recv=31860 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=34816 expected_recv=32559 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=34816 expected_recv=32442 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=34816 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=34816 expected_recv=32802 expected_overflow=0 exact_recv_per_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=34816 expected_recv=31083 expected_overflow=0 exact_recv_per_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=34816 expected_recv=32779 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=34816 expected_recv=32802 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=34816 expected_recv=31083 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=31488 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=34816 expected_recv=31798 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=34816 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=34816 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=34816 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=34816 expected_recv=31798 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=34816 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=34816 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=34816 expected_recv=32120 expected_overflow=0 exact_recv_per_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=34816 expected_recv=32772 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=34816 expected_recv=32749 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=34816 expected_recv=32770 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=34816 expected_recv=32120 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=34816 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=34816 expected_recv=32636 expected_overflow=0 exact_recv_per_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=34816 expected_recv=32767 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=34816 expected_recv=32977 expected_overflow=0 exact_recv_per_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=34816 expected_recv=32354 expected_overflow=0 exact_recv_per_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=34816 expected_recv=32467 expected_overflow=0 exact_recv_per_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=34816 expected_recv=32636 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=34816 expected_recv=32977 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=34816 expected_recv=32354 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=34816 expected_recv=32467 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=34816 expected_recv=31995 expected_overflow=0 exact_recv_per_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=34816 expected_recv=31995 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=34816 expected_recv=33023 expected_overflow=0 exact_recv_per_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=34816 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=34816 expected_recv=32605 expected_overflow=0 exact_recv_per_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=34816 expected_recv=33023 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=34816 expected_recv=32651 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=34816 expected_recv=32605 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=34816 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=34816 expected_recv=31899 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=34816 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=34816 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=34816 expected_recv=32766 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=34816 expected_recv=31899 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=34816 expected_recv=32773 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=34816 expected_recv=32785 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32000 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=34816 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=34816 expected_recv=32721 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=34816 expected_recv=31486 expected_overflow=0 exact_recv_per_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=34816 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=34816 expected_recv=31486 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[39389, 40913, 32550, 27908, 33662, 23525, 27432, 36765] avg=32768.0000 max=40913 max_rank=1 min=23525 min_rank=5 imbalance_ratio=1.248566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[39389, 40913, 32550, 27908, 33662, 23525, 27432, 36765] avg=32768.0000 max=40913 max_rank=1 min=23525 min_rank=5 imbalance_ratio=1.248566 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=34816 expected_recv=32768 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=31744 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[39389, 40913, 32550, 27908, 33662, 23525, 27432, 36765] avg=32768.0000 max=40913 max_rank=1 min=23525 min_rank=5 imbalance_ratio=1.248566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] avg=32768.0000 max=33097 max_rank=1 min=31907 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=7816 max_load_reduction_pct=0.191040 moved_to_replicas=18345 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] avg=32768.0000 max=33097 max_rank=1 min=31907 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=7816 max_load_reduction_pct=0.191040 moved_to_replicas=18345 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[39389, 40913, 32550, 27908, 33662, 23525, 27432, 36765] avg=32768.0000 max=40913 max_rank=1 min=23525 min_rank=5 imbalance_ratio=1.248566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] avg=32768.0000 max=33097 max_rank=1 min=31907 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=7816 max_load_reduction_pct=0.191040 moved_to_replicas=18345 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[39389, 40913, 32550, 27908, 33662, 23525, 27432, 36765] avg=32768.0000 max=40913 max_rank=1 min=23525 min_rank=5 imbalance_ratio=1.248566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[39389, 40913, 32550, 27908, 33662, 23525, 27432, 36765] avg=32768.0000 max=40913 max_rank=1 min=23525 min_rank=5 imbalance_ratio=1.248566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[39389, 40913, 32550, 27908, 33662, 23525, 27432, 36765] avg=32768.0000 max=40913 max_rank=1 min=23525 min_rank=5 imbalance_ratio=1.248566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[39389, 40913, 32550, 27908, 33662, 23525, 27432, 36765] avg=32768.0000 max=40913 max_rank=1 min=23525 min_rank=5 imbalance_ratio=1.248566 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[56708, 67303, 32517, 44585, 17877, 3099, 11875, 28180] avg=32768.0000 max=67303 max_rank=1 min=3099 min_rank=5 imbalance_ratio=2.053925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[56708, 67303, 32517, 44585, 17877, 3099, 11875, 28180] avg=32768.0000 max=67303 max_rank=1 min=3099 min_rank=5 imbalance_ratio=2.053925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] avg=32768.0000 max=33097 max_rank=1 min=31907 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=7816 max_load_reduction_pct=0.191040 moved_to_replicas=18345 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] avg=32768.0000 max=33097 max_rank=1 min=31907 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=7816 max_load_reduction_pct=0.191040 moved_to_replicas=18345 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] avg=32768.0000 max=33097 max_rank=1 min=31907 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=7816 max_load_reduction_pct=0.191040 moved_to_replicas=18345 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[56708, 67303, 32517, 44585, 17877, 3099, 11875, 28180] avg=32768.0000 max=67303 max_rank=1 min=3099 min_rank=5 imbalance_ratio=2.053925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] avg=32768.0000 max=33097 max_rank=1 min=31907 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=7816 max_load_reduction_pct=0.191040 moved_to_replicas=18345 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] avg=32768.0000 max=33097 max_rank=4 min=31050 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34206 max_load_reduction_pct=0.508239 moved_to_replicas=69308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[33095, 33097, 32550, 32205, 33096, 31907, 33097, 33097] avg=32768.0000 max=33097 max_rank=1 min=31907 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=7816 max_load_reduction_pct=0.191040 moved_to_replicas=18345 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] avg=32768.0000 max=33097 max_rank=4 min=31050 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34206 max_load_reduction_pct=0.508239 moved_to_replicas=69308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[56708, 67303, 32517, 44585, 17877, 3099, 11875, 28180] avg=32768.0000 max=67303 max_rank=1 min=3099 min_rank=5 imbalance_ratio=2.053925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[56708, 67303, 32517, 44585, 17877, 3099, 11875, 28180] avg=32768.0000 max=67303 max_rank=1 min=3099 min_rank=5 imbalance_ratio=2.053925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] avg=32768.0000 max=33097 max_rank=4 min=31050 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34206 max_load_reduction_pct=0.508239 moved_to_replicas=69308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[56708, 67303, 32517, 44585, 17877, 3099, 11875, 28180] avg=32768.0000 max=67303 max_rank=1 min=3099 min_rank=5 imbalance_ratio=2.053925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[16806, 30949, 64071, 81386, 36793, 30966, 9, 1164] avg=32768.0000 max=81386 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.483704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[56708, 67303, 32517, 44585, 17877, 3099, 11875, 28180] avg=32768.0000 max=67303 max_rank=1 min=3099 min_rank=5 imbalance_ratio=2.053925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[16806, 30949, 64071, 81386, 36793, 30966, 9, 1164] avg=32768.0000 max=81386 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.483704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] avg=32768.0000 max=33097 max_rank=4 min=31050 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34206 max_load_reduction_pct=0.508239 moved_to_replicas=69308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[56708, 67303, 32517, 44585, 17877, 3099, 11875, 28180] avg=32768.0000 max=67303 max_rank=1 min=3099 min_rank=5 imbalance_ratio=2.053925 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] avg=32768.0000 max=33097 max_rank=4 min=31050 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34206 max_load_reduction_pct=0.508239 moved_to_replicas=69308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] avg=32768.0000 max=33097 max_rank=4 min=31050 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34206 max_load_reduction_pct=0.508239 moved_to_replicas=69308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] avg=32768.0000 max=33096 max_rank=0 min=31705 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=48290 max_load_reduction_pct=0.593345 moved_to_replicas=82962 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[16806, 30949, 64071, 81386, 36793, 30966, 9, 1164] avg=32768.0000 max=81386 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.483704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] avg=32768.0000 max=33097 max_rank=4 min=31050 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34206 max_load_reduction_pct=0.508239 moved_to_replicas=69308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] avg=32768.0000 max=33096 max_rank=0 min=31705 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=48290 max_load_reduction_pct=0.593345 moved_to_replicas=82962 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[33096, 33096, 32517, 33095, 33097, 33097, 31050, 33096] avg=32768.0000 max=33097 max_rank=4 min=31050 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=34206 max_load_reduction_pct=0.508239 moved_to_replicas=69308 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[16806, 30949, 64071, 81386, 36793, 30966, 9, 1164] avg=32768.0000 max=81386 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.483704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[16806, 30949, 64071, 81386, 36793, 30966, 9, 1164] avg=32768.0000 max=81386 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.483704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] avg=32768.0000 max=33096 max_rank=0 min=31705 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=48290 max_load_reduction_pct=0.593345 moved_to_replicas=82962 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[16806, 30949, 64071, 81386, 36793, 30966, 9, 1164] avg=32768.0000 max=81386 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.483704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[54375, 1197, 23220, 32442, 22787, 160, 10330, 117633] avg=32768.0000 max=117633 max_rank=7 min=160 min_rank=5 imbalance_ratio=3.589874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[16806, 30949, 64071, 81386, 36793, 30966, 9, 1164] avg=32768.0000 max=81386 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.483704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[54375, 1197, 23220, 32442, 22787, 160, 10330, 117633] avg=32768.0000 max=117633 max_rank=7 min=160 min_rank=5 imbalance_ratio=3.589874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] avg=32768.0000 max=33096 max_rank=0 min=31705 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=48290 max_load_reduction_pct=0.593345 moved_to_replicas=82962 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[16806, 30949, 64071, 81386, 36793, 30966, 9, 1164] avg=32768.0000 max=81386 max_rank=3 min=9 min_rank=6 imbalance_ratio=2.483704 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] avg=32768.0000 max=33096 max_rank=0 min=31705 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=48290 max_load_reduction_pct=0.593345 moved_to_replicas=82962 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] avg=32768.0000 max=33097 max_rank=2 min=31860 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=84536 max_load_reduction_pct=0.718642 moved_to_replicas=105816 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] avg=32768.0000 max=33096 max_rank=0 min=31705 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=48290 max_load_reduction_pct=0.593345 moved_to_replicas=82962 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[54375, 1197, 23220, 32442, 22787, 160, 10330, 117633] avg=32768.0000 max=117633 max_rank=7 min=160 min_rank=5 imbalance_ratio=3.589874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] avg=32768.0000 max=33096 max_rank=0 min=31705 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=48290 max_load_reduction_pct=0.593345 moved_to_replicas=82962 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] avg=32768.0000 max=33097 max_rank=2 min=31860 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=84536 max_load_reduction_pct=0.718642 moved_to_replicas=105816 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[33096, 33096, 33096, 33096, 33096, 32516, 32443, 31705] avg=32768.0000 max=33096 max_rank=0 min=31705 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=48290 max_load_reduction_pct=0.593345 moved_to_replicas=82962 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[54375, 1197, 23220, 32442, 22787, 160, 10330, 117633] avg=32768.0000 max=117633 max_rank=7 min=160 min_rank=5 imbalance_ratio=3.589874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[54375, 1197, 23220, 32442, 22787, 160, 10330, 117633] avg=32768.0000 max=117633 max_rank=7 min=160 min_rank=5 imbalance_ratio=3.589874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] avg=32768.0000 max=33097 max_rank=2 min=31860 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=84536 max_load_reduction_pct=0.718642 moved_to_replicas=105816 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[12479, 76, 12, 19003, 106777, 88519, 33171, 2107] avg=32768.0000 max=106777 max_rank=4 min=12 min_rank=2 imbalance_ratio=3.258575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[54375, 1197, 23220, 32442, 22787, 160, 10330, 117633] avg=32768.0000 max=117633 max_rank=7 min=160 min_rank=5 imbalance_ratio=3.589874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[12479, 76, 12, 19003, 106777, 88519, 33171, 2107] avg=32768.0000 max=106777 max_rank=4 min=12 min_rank=2 imbalance_ratio=3.258575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[54375, 1197, 23220, 32442, 22787, 160, 10330, 117633] avg=32768.0000 max=117633 max_rank=7 min=160 min_rank=5 imbalance_ratio=3.589874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] avg=32768.0000 max=33097 max_rank=2 min=31860 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=84536 max_load_reduction_pct=0.718642 moved_to_replicas=105816 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] avg=32768.0000 max=33097 max_rank=2 min=31860 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=84536 max_load_reduction_pct=0.718642 moved_to_replicas=105816 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[54375, 1197, 23220, 32442, 22787, 160, 10330, 117633] avg=32768.0000 max=117633 max_rank=7 min=160 min_rank=5 imbalance_ratio=3.589874 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31083 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73680 max_load_reduction_pct=0.690036 moved_to_replicas=129179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] avg=32768.0000 max=33097 max_rank=2 min=31860 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=84536 max_load_reduction_pct=0.718642 moved_to_replicas=105816 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[12479, 76, 12, 19003, 106777, 88519, 33171, 2107] avg=32768.0000 max=106777 max_rank=4 min=12 min_rank=2 imbalance_ratio=3.258575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31083 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73680 max_load_reduction_pct=0.690036 moved_to_replicas=129179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] avg=32768.0000 max=33097 max_rank=2 min=31860 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=84536 max_load_reduction_pct=0.718642 moved_to_replicas=105816 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[33095, 32559, 33097, 32442, 33096, 32900, 31860, 33095] avg=32768.0000 max=33097 max_rank=2 min=31860 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=84536 max_load_reduction_pct=0.718642 moved_to_replicas=105816 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[12479, 76, 12, 19003, 106777, 88519, 33171, 2107] avg=32768.0000 max=106777 max_rank=4 min=12 min_rank=2 imbalance_ratio=3.258575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[12479, 76, 12, 19003, 106777, 88519, 33171, 2107] avg=32768.0000 max=106777 max_rank=4 min=12 min_rank=2 imbalance_ratio=3.258575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[42302, 32749, 32772, 32770, 31621, 36461, 32767, 20702] avg=32768.0000 max=42302 max_rank=0 min=20702 min_rank=7 imbalance_ratio=1.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31083 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73680 max_load_reduction_pct=0.690036 moved_to_replicas=129179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[12479, 76, 12, 19003, 106777, 88519, 33171, 2107] avg=32768.0000 max=106777 max_rank=4 min=12 min_rank=2 imbalance_ratio=3.258575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[42302, 32749, 32772, 32770, 31621, 36461, 32767, 20702] avg=32768.0000 max=42302 max_rank=0 min=20702 min_rank=7 imbalance_ratio=1.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31083 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73680 max_load_reduction_pct=0.690036 moved_to_replicas=129179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[12479, 76, 12, 19003, 106777, 88519, 33171, 2107] avg=32768.0000 max=106777 max_rank=4 min=12 min_rank=2 imbalance_ratio=3.258575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31083 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73680 max_load_reduction_pct=0.690036 moved_to_replicas=129179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31798 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=9206 max_load_reduction_pct=0.217626 moved_to_replicas=12571 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[12479, 76, 12, 19003, 106777, 88519, 33171, 2107] avg=32768.0000 max=106777 max_rank=4 min=12 min_rank=2 imbalance_ratio=3.258575 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31083 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73680 max_load_reduction_pct=0.690036 moved_to_replicas=129179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31798 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=9206 max_load_reduction_pct=0.217626 moved_to_replicas=12571 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[42302, 32749, 32772, 32770, 31621, 36461, 32767, 20702] avg=32768.0000 max=42302 max_rank=0 min=20702 min_rank=7 imbalance_ratio=1.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31083 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73680 max_load_reduction_pct=0.690036 moved_to_replicas=129179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[33096, 32802, 32779, 31083, 33096, 33097, 33096, 33095] avg=32768.0000 max=33097 max_rank=5 min=31083 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=73680 max_load_reduction_pct=0.690036 moved_to_replicas=129179 active_clone_links=8 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[42302, 32749, 32772, 32770, 31621, 36461, 32767, 20702] avg=32768.0000 max=42302 max_rank=0 min=20702 min_rank=7 imbalance_ratio=1.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[42302, 32749, 32772, 32770, 31621, 36461, 32767, 20702] avg=32768.0000 max=42302 max_rank=0 min=20702 min_rank=7 imbalance_ratio=1.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31798 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=9206 max_load_reduction_pct=0.217626 moved_to_replicas=12571 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[13568, 71, 581, 32977, 0, 65142, 117685, 32120] avg=32768.0000 max=117685 max_rank=6 min=0 min_rank=4 imbalance_ratio=3.591461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[42302, 32749, 32772, 32770, 31621, 36461, 32767, 20702] avg=32768.0000 max=42302 max_rank=0 min=20702 min_rank=7 imbalance_ratio=1.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[13568, 71, 581, 32977, 0, 65142, 117685, 32120] avg=32768.0000 max=117685 max_rank=6 min=0 min_rank=4 imbalance_ratio=3.591461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31798 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=9206 max_load_reduction_pct=0.217626 moved_to_replicas=12571 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[42302, 32749, 32772, 32770, 31621, 36461, 32767, 20702] avg=32768.0000 max=42302 max_rank=0 min=20702 min_rank=7 imbalance_ratio=1.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31798 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=9206 max_load_reduction_pct=0.217626 moved_to_replicas=12571 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[42302, 32749, 32772, 32770, 31621, 36461, 32767, 20702] avg=32768.0000 max=42302 max_rank=0 min=20702 min_rank=7 imbalance_ratio=1.290955 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] avg=32768.0000 max=33097 max_rank=0 min=32120 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84588 max_load_reduction_pct=0.718766 moved_to_replicas=116635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31798 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=9206 max_load_reduction_pct=0.217626 moved_to_replicas=12571 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[13568, 71, 581, 32977, 0, 65142, 117685, 32120] avg=32768.0000 max=117685 max_rank=6 min=0 min_rank=4 imbalance_ratio=3.591461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] avg=32768.0000 max=33097 max_rank=0 min=32120 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84588 max_load_reduction_pct=0.718766 moved_to_replicas=116635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31798 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=9206 max_load_reduction_pct=0.217626 moved_to_replicas=12571 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[33096, 32749, 32772, 32770, 31798, 33096, 32767, 33096] avg=32768.0000 max=33096 max_rank=0 min=31798 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=9206 max_load_reduction_pct=0.217626 moved_to_replicas=12571 active_clone_links=3 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[13568, 71, 581, 32977, 0, 65142, 117685, 32120] avg=32768.0000 max=117685 max_rank=6 min=0 min_rank=4 imbalance_ratio=3.591461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] avg=32768.0000 max=33097 max_rank=0 min=32120 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84588 max_load_reduction_pct=0.718766 moved_to_replicas=116635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[32605, 22649, 21674, 79717, 31995, 41036, 1, 32467] avg=32768.0000 max=79717 max_rank=3 min=1 min_rank=6 imbalance_ratio=2.432770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[13568, 71, 581, 32977, 0, 65142, 117685, 32120] avg=32768.0000 max=117685 max_rank=6 min=0 min_rank=4 imbalance_ratio=3.591461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[13568, 71, 581, 32977, 0, 65142, 117685, 32120] avg=32768.0000 max=117685 max_rank=6 min=0 min_rank=4 imbalance_ratio=3.591461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[32605, 22649, 21674, 79717, 31995, 41036, 1, 32467] avg=32768.0000 max=79717 max_rank=3 min=1 min_rank=6 imbalance_ratio=2.432770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] avg=32768.0000 max=33097 max_rank=0 min=32120 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84588 max_load_reduction_pct=0.718766 moved_to_replicas=116635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[13568, 71, 581, 32977, 0, 65142, 117685, 32120] avg=32768.0000 max=117685 max_rank=6 min=0 min_rank=4 imbalance_ratio=3.591461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46621 max_load_reduction_pct=0.584831 moved_to_replicas=54561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[13568, 71, 581, 32977, 0, 65142, 117685, 32120] avg=32768.0000 max=117685 max_rank=6 min=0 min_rank=4 imbalance_ratio=3.591461 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] avg=32768.0000 max=33097 max_rank=0 min=32120 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84588 max_load_reduction_pct=0.718766 moved_to_replicas=116635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] avg=32768.0000 max=33097 max_rank=0 min=32120 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84588 max_load_reduction_pct=0.718766 moved_to_replicas=116635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46621 max_load_reduction_pct=0.584831 moved_to_replicas=54561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[32605, 22649, 21674, 79717, 31995, 41036, 1, 32467] avg=32768.0000 max=79717 max_rank=3 min=1 min_rank=6 imbalance_ratio=2.432770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] avg=32768.0000 max=33097 max_rank=0 min=32120 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84588 max_load_reduction_pct=0.718766 moved_to_replicas=116635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[33097, 32636, 32354, 32977, 32767, 33096, 33097, 32120] avg=32768.0000 max=33097 max_rank=0 min=32120 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=84588 max_load_reduction_pct=0.718766 moved_to_replicas=116635 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[32605, 22649, 21674, 79717, 31995, 41036, 1, 32467] avg=32768.0000 max=79717 max_rank=3 min=1 min_rank=6 imbalance_ratio=2.432770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[3347, 32766, 32773, 31899, 1, 72802, 65598, 22958] avg=32768.0000 max=72802 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.221741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46621 max_load_reduction_pct=0.584831 moved_to_replicas=54561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[32605, 22649, 21674, 79717, 31995, 41036, 1, 32467] avg=32768.0000 max=79717 max_rank=3 min=1 min_rank=6 imbalance_ratio=2.432770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[32605, 22649, 21674, 79717, 31995, 41036, 1, 32467] avg=32768.0000 max=79717 max_rank=3 min=1 min_rank=6 imbalance_ratio=2.432770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[3347, 32766, 32773, 31899, 1, 72802, 65598, 22958] avg=32768.0000 max=72802 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.221741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46621 max_load_reduction_pct=0.584831 moved_to_replicas=54561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39706 max_load_reduction_pct=0.545397 moved_to_replicas=72208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[32605, 22649, 21674, 79717, 31995, 41036, 1, 32467] avg=32768.0000 max=79717 max_rank=3 min=1 min_rank=6 imbalance_ratio=2.432770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[32605, 22649, 21674, 79717, 31995, 41036, 1, 32467] avg=32768.0000 max=79717 max_rank=3 min=1 min_rank=6 imbalance_ratio=2.432770 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46621 max_load_reduction_pct=0.584831 moved_to_replicas=54561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46621 max_load_reduction_pct=0.584831 moved_to_replicas=54561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39706 max_load_reduction_pct=0.545397 moved_to_replicas=72208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[3347, 32766, 32773, 31899, 1, 72802, 65598, 22958] avg=32768.0000 max=72802 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.221741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46621 max_load_reduction_pct=0.584831 moved_to_replicas=54561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[32605, 33023, 33096, 33096, 31995, 33094, 32768, 32467] avg=32768.0000 max=33096 max_rank=2 min=31995 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=46621 max_load_reduction_pct=0.584831 moved_to_replicas=54561 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[3347, 32766, 32773, 31899, 1, 72802, 65598, 22958] avg=32768.0000 max=72802 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.221741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[62796, 29967, 32768, 33140, 32721, 32785, 37967, 0] avg=32768.0000 max=62796 max_rank=0 min=0 min_rank=7 imbalance_ratio=1.916382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39706 max_load_reduction_pct=0.545397 moved_to_replicas=72208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[3347, 32766, 32773, 31899, 1, 72802, 65598, 22958] avg=32768.0000 max=72802 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.221741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[62796, 29967, 32768, 33140, 32721, 32785, 37967, 0] avg=32768.0000 max=62796 max_rank=0 min=0 min_rank=7 imbalance_ratio=1.916382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[3347, 32766, 32773, 31899, 1, 72802, 65598, 22958] avg=32768.0000 max=72802 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.221741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39706 max_load_reduction_pct=0.545397 moved_to_replicas=72208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29700 max_load_reduction_pct=0.472960 moved_to_replicas=34615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[3347, 32766, 32773, 31899, 1, 72802, 65598, 22958] avg=32768.0000 max=72802 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.221741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[3347, 32766, 32773, 31899, 1, 72802, 65598, 22958] avg=32768.0000 max=72802 max_rank=5 min=1 min_rank=4 imbalance_ratio=2.221741 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39706 max_load_reduction_pct=0.545397 moved_to_replicas=72208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29700 max_load_reduction_pct=0.472960 moved_to_replicas=34615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[62796, 29967, 32768, 33140, 32721, 32785, 37967, 0] avg=32768.0000 max=62796 max_rank=0 min=0 min_rank=7 imbalance_ratio=1.916382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39706 max_load_reduction_pct=0.545397 moved_to_replicas=72208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39706 max_load_reduction_pct=0.545397 moved_to_replicas=72208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[33096, 32766, 32773, 31899, 32768, 33095, 33096, 32651] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=39706 max_load_reduction_pct=0.545397 moved_to_replicas=72208 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[62796, 29967, 32768, 33140, 32721, 32785, 37967, 0] avg=32768.0000 max=62796 max_rank=0 min=0 min_rank=7 imbalance_ratio=1.916382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29700 max_load_reduction_pct=0.472960 moved_to_replicas=34615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[62796, 29967, 32768, 33140, 32721, 32785, 37967, 0] avg=32768.0000 max=62796 max_rank=0 min=0 min_rank=7 imbalance_ratio=1.916382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[62796, 29967, 32768, 33140, 32721, 32785, 37967, 0] avg=32768.0000 max=62796 max_rank=0 min=0 min_rank=7 imbalance_ratio=1.916382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29700 max_load_reduction_pct=0.472960 moved_to_replicas=34615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[62796, 29967, 32768, 33140, 32721, 32785, 37967, 0] avg=32768.0000 max=62796 max_rank=0 min=0 min_rank=7 imbalance_ratio=1.916382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[62796, 29967, 32768, 33140, 32721, 32785, 37967, 0] avg=32768.0000 max=62796 max_rank=0 min=0 min_rank=7 imbalance_ratio=1.916382 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29700 max_load_reduction_pct=0.472960 moved_to_replicas=34615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29700 max_load_reduction_pct=0.472960 moved_to_replicas=34615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29700 max_load_reduction_pct=0.472960 moved_to_replicas=34615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[33096, 31486, 32768, 33096, 32721, 32785, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31486 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=29700 max_load_reduction_pct=0.472960 moved_to_replicas=34615 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=10 total_expected_recv=327518 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=17024 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=10 total_expected_recv=328683 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=15872 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=10 total_expected_recv=327530 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=17792 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=10 total_expected_recv=327310 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=18048 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=10 total_expected_recv=330468 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=14336 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=10 total_expected_recv=326370 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=17792 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=10 total_expected_recv=325759 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=19328 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=10 total_expected_recv=327802 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=17152 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 18:02:39] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 244.9 | throughput per GPU (TFLOP/s/GPU): 504.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.698555E+00 | loss scale: 1.0 | grad norm: 4.444 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=34816 expected_recv=31920 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=34816 expected_recv=31940 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=34816 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=34816 expected_recv=31920 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=34816 expected_recv=31940 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=34816 expected_recv=32805 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=32384 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=34816 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=34816 expected_recv=32708 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=34816 expected_recv=30858 expected_overflow=0 exact_recv_per_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=34048 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=34048 slack_slots=768 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=34816 expected_recv=30858 actual_valid_recv=31232 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=31232 slack_slots=3584 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=34816 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=34816 expected_recv=31000 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=34816 expected_recv=32568 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=34816 expected_recv=31000 actual_valid_recv=31104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=31104 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=34816 expected_recv=32204 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=34816 expected_recv=31414 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=34816 expected_recv=33045 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=34816 expected_recv=32204 actual_valid_recv=32640 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=34816 expected_recv=31414 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=34816 expected_recv=33045 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=32640 slack_slots=2176 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=31488 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=34816 expected_recv=31122 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=34816 expected_recv=31122 actual_valid_recv=31360 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=31360 slack_slots=3456 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=34816 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=34816 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=34816 expected_recv=32781 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=34816 expected_recv=32762 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=34816 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=34816 expected_recv=33098 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=34816 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=34816 expected_recv=32769 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=34816 expected_recv=31815 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=34816 expected_recv=31815 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=34816 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=34816 expected_recv=32079 expected_overflow=0 exact_recv_per_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=34816 expected_recv=33093 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=34816 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=34816 expected_recv=32079 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=34816 expected_recv=33093 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=34816 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=34816 expected_recv=32772 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=34816 expected_recv=31089 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=34816 expected_recv=31089 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=34816 expected_recv=30783 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=34816 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=31488 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=34816 expected_recv=30783 actual_valid_recv=31104 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=34816 expected_recv=32805 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=31104 slack_slots=3712 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=34816 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=34816 expected_recv=32787 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=34816 expected_recv=32277 expected_overflow=0 exact_recv_per_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=34816 expected_recv=32277 actual_valid_recv=32512 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=34816 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32512 slack_slots=2304 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=34816 expected_recv=32790 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=34816 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=34816 expected_recv=32769 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=34816 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=34816 expected_recv=32022 expected_overflow=0 exact_recv_per_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=34816 expected_recv=32246 expected_overflow=0 exact_recv_per_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=34816 expected_recv=32775 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=34816 expected_recv=32022 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=34816 expected_recv=32246 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=34816 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=34816 expected_recv=32745 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=34816 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=34816 expected_recv=32210 expected_overflow=0 exact_recv_per_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=34816 expected_recv=32780 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[37579, 35932, 31348, 29211, 33647, 26369, 31369, 36689] avg=32768.0000 max=37579 max_rank=0 min=26369 min_rank=5 imbalance_ratio=1.146820 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=34816 expected_recv=33098 expected_overflow=0 exact_recv_per_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=34816 expected_recv=32210 actual_valid_recv=32384 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32896 slack_slots=1920 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=34816 expected_recv=33098 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32384 slack_slots=2432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[37579, 35932, 31348, 29211, 33647, 26369, 31369, 36689] avg=32768.0000 max=37579 max_rank=0 min=26369 min_rank=5 imbalance_ratio=1.146820 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] avg=32768.0000 max=33097 max_rank=0 min=31920 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4482 max_load_reduction_pct=0.119269 moved_to_replicas=11463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[37579, 35932, 31348, 29211, 33647, 26369, 31369, 36689] avg=32768.0000 max=37579 max_rank=0 min=26369 min_rank=5 imbalance_ratio=1.146820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] avg=32768.0000 max=33097 max_rank=0 min=31920 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4482 max_load_reduction_pct=0.119269 moved_to_replicas=11463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[37579, 35932, 31348, 29211, 33647, 26369, 31369, 36689] avg=32768.0000 max=37579 max_rank=0 min=26369 min_rank=5 imbalance_ratio=1.146820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[37579, 35932, 31348, 29211, 33647, 26369, 31369, 36689] avg=32768.0000 max=37579 max_rank=0 min=26369 min_rank=5 imbalance_ratio=1.146820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[37579, 35932, 31348, 29211, 33647, 26369, 31369, 36689] avg=32768.0000 max=37579 max_rank=0 min=26369 min_rank=5 imbalance_ratio=1.146820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] avg=32768.0000 max=33097 max_rank=0 min=31920 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4482 max_load_reduction_pct=0.119269 moved_to_replicas=11463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[37579, 35932, 31348, 29211, 33647, 26369, 31369, 36689] avg=32768.0000 max=37579 max_rank=0 min=26369 min_rank=5 imbalance_ratio=1.146820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[30858, 55771, 16557, 10777, 19176, 18819, 50635, 59551] avg=32768.0000 max=59551 max_rank=7 min=10777 min_rank=3 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[37579, 35932, 31348, 29211, 33647, 26369, 31369, 36689] avg=32768.0000 max=37579 max_rank=0 min=26369 min_rank=5 imbalance_ratio=1.146820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] avg=32768.0000 max=33097 max_rank=0 min=31920 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4482 max_load_reduction_pct=0.119269 moved_to_replicas=11463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] avg=32768.0000 max=33097 max_rank=0 min=31920 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4482 max_load_reduction_pct=0.119269 moved_to_replicas=11463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] avg=32768.0000 max=33097 max_rank=0 min=31920 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4482 max_load_reduction_pct=0.119269 moved_to_replicas=11463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[30858, 55771, 16557, 10777, 19176, 18819, 50635, 59551] avg=32768.0000 max=59551 max_rank=7 min=10777 min_rank=3 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=66669 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] avg=32768.0000 max=33097 max_rank=0 min=31920 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4482 max_load_reduction_pct=0.119269 moved_to_replicas=11463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[30858, 55771, 16557, 10777, 19176, 18819, 50635, 59551] avg=32768.0000 max=59551 max_rank=7 min=10777 min_rank=3 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[33097, 33097, 31940, 32805, 33096, 33094, 31920, 33095] avg=32768.0000 max=33097 max_rank=0 min=31920 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=4482 max_load_reduction_pct=0.119269 moved_to_replicas=11463 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[30858, 55771, 16557, 10777, 19176, 18819, 50635, 59551] avg=32768.0000 max=59551 max_rank=7 min=10777 min_rank=3 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=66669 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=66669 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[30858, 55771, 16557, 10777, 19176, 18819, 50635, 59551] avg=32768.0000 max=59551 max_rank=7 min=10777 min_rank=3 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[30858, 55771, 16557, 10777, 19176, 18819, 50635, 59551] avg=32768.0000 max=59551 max_rank=7 min=10777 min_rank=3 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[19532, 15817, 57618, 31000, 46965, 30557, 14, 60641] avg=32768.0000 max=60641 max_rank=7 min=14 min_rank=6 imbalance_ratio=1.850616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[30858, 55771, 16557, 10777, 19176, 18819, 50635, 59551] avg=32768.0000 max=59551 max_rank=7 min=10777 min_rank=3 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[30858, 55771, 16557, 10777, 19176, 18819, 50635, 59551] avg=32768.0000 max=59551 max_rank=7 min=10777 min_rank=3 imbalance_ratio=1.817352 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=66669 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=66669 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=66669 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[19532, 15817, 57618, 31000, 46965, 30557, 14, 60641] avg=32768.0000 max=60641 max_rank=7 min=14 min_rank=6 imbalance_ratio=1.850616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[19532, 15817, 57618, 31000, 46965, 30557, 14, 60641] avg=32768.0000 max=60641 max_rank=7 min=14 min_rank=6 imbalance_ratio=1.850616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31000 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27544 max_load_reduction_pct=0.454214 moved_to_replicas=65936 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=66669 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[30858, 33096, 33097, 33096, 32708, 33096, 33097, 33096] avg=32768.0000 max=33097 max_rank=2 min=30858 min_rank=0 imbalance_ratio=1.010040 max_load_reduction=26454 max_load_reduction_pct=0.444224 moved_to_replicas=66669 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[19532, 15817, 57618, 31000, 46965, 30557, 14, 60641] avg=32768.0000 max=60641 max_rank=7 min=14 min_rank=6 imbalance_ratio=1.850616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31000 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27544 max_load_reduction_pct=0.454214 moved_to_replicas=65936 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31000 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27544 max_load_reduction_pct=0.454214 moved_to_replicas=65936 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[19532, 15817, 57618, 31000, 46965, 30557, 14, 60641] avg=32768.0000 max=60641 max_rank=7 min=14 min_rank=6 imbalance_ratio=1.850616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[19532, 15817, 57618, 31000, 46965, 30557, 14, 60641] avg=32768.0000 max=60641 max_rank=7 min=14 min_rank=6 imbalance_ratio=1.850616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[44238, 5870, 10739, 37036, 35669, 21063, 31414, 76115] avg=32768.0000 max=76115 max_rank=7 min=5870 min_rank=1 imbalance_ratio=2.322845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[19532, 15817, 57618, 31000, 46965, 30557, 14, 60641] avg=32768.0000 max=60641 max_rank=7 min=14 min_rank=6 imbalance_ratio=1.850616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[19532, 15817, 57618, 31000, 46965, 30557, 14, 60641] avg=32768.0000 max=60641 max_rank=7 min=14 min_rank=6 imbalance_ratio=1.850616 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31000 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27544 max_load_reduction_pct=0.454214 moved_to_replicas=65936 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31000 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27544 max_load_reduction_pct=0.454214 moved_to_replicas=65936 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31000 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27544 max_load_reduction_pct=0.454214 moved_to_replicas=65936 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] avg=32768.0000 max=33097 max_rank=0 min=31414 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43018 max_load_reduction_pct=0.565171 moved_to_replicas=60674 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[44238, 5870, 10739, 37036, 35669, 21063, 31414, 76115] avg=32768.0000 max=76115 max_rank=7 min=5870 min_rank=1 imbalance_ratio=2.322845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[44238, 5870, 10739, 37036, 35669, 21063, 31414, 76115] avg=32768.0000 max=76115 max_rank=7 min=5870 min_rank=1 imbalance_ratio=2.322845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31000 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27544 max_load_reduction_pct=0.454214 moved_to_replicas=65936 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[33096, 33095, 33097, 31000, 33096, 32568, 33096, 33096] avg=32768.0000 max=33097 max_rank=2 min=31000 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=27544 max_load_reduction_pct=0.454214 moved_to_replicas=65936 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] avg=32768.0000 max=33097 max_rank=0 min=31414 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43018 max_load_reduction_pct=0.565171 moved_to_replicas=60674 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] avg=32768.0000 max=33097 max_rank=0 min=31414 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43018 max_load_reduction_pct=0.565171 moved_to_replicas=60674 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[44238, 5870, 10739, 37036, 35669, 21063, 31414, 76115] avg=32768.0000 max=76115 max_rank=7 min=5870 min_rank=1 imbalance_ratio=2.322845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[44238, 5870, 10739, 37036, 35669, 21063, 31414, 76115] avg=32768.0000 max=76115 max_rank=7 min=5870 min_rank=1 imbalance_ratio=2.322845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[44238, 5870, 10739, 37036, 35669, 21063, 31414, 76115] avg=32768.0000 max=76115 max_rank=7 min=5870 min_rank=1 imbalance_ratio=2.322845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[10932, 98, 14, 9479, 100249, 61466, 61319, 18587] avg=32768.0000 max=100249 max_rank=4 min=14 min_rank=2 imbalance_ratio=3.059357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] avg=32768.0000 max=33097 max_rank=0 min=31414 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43018 max_load_reduction_pct=0.565171 moved_to_replicas=60674 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[44238, 5870, 10739, 37036, 35669, 21063, 31414, 76115] avg=32768.0000 max=76115 max_rank=7 min=5870 min_rank=1 imbalance_ratio=2.322845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[44238, 5870, 10739, 37036, 35669, 21063, 31414, 76115] avg=32768.0000 max=76115 max_rank=7 min=5870 min_rank=1 imbalance_ratio=2.322845 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] avg=32768.0000 max=33097 max_rank=0 min=31414 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43018 max_load_reduction_pct=0.565171 moved_to_replicas=60674 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] avg=32768.0000 max=33097 max_rank=0 min=31414 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43018 max_load_reduction_pct=0.565171 moved_to_replicas=60674 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] avg=32768.0000 max=33097 max_rank=3 min=31122 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67152 max_load_reduction_pct=0.669852 moved_to_replicas=123746 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[10932, 98, 14, 9479, 100249, 61466, 61319, 18587] avg=32768.0000 max=100249 max_rank=4 min=14 min_rank=2 imbalance_ratio=3.059357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[10932, 98, 14, 9479, 100249, 61466, 61319, 18587] avg=32768.0000 max=100249 max_rank=4 min=14 min_rank=2 imbalance_ratio=3.059357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] avg=32768.0000 max=33097 max_rank=0 min=31414 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43018 max_load_reduction_pct=0.565171 moved_to_replicas=60674 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[33097, 33096, 33045, 33096, 33096, 32204, 31414, 33096] avg=32768.0000 max=33097 max_rank=0 min=31414 min_rank=6 imbalance_ratio=1.010040 max_load_reduction=43018 max_load_reduction_pct=0.565171 moved_to_replicas=60674 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] avg=32768.0000 max=33097 max_rank=3 min=31122 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67152 max_load_reduction_pct=0.669852 moved_to_replicas=123746 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[10932, 98, 14, 9479, 100249, 61466, 61319, 18587] avg=32768.0000 max=100249 max_rank=4 min=14 min_rank=2 imbalance_ratio=3.059357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] avg=32768.0000 max=33097 max_rank=3 min=31122 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67152 max_load_reduction_pct=0.669852 moved_to_replicas=123746 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[10932, 98, 14, 9479, 100249, 61466, 61319, 18587] avg=32768.0000 max=100249 max_rank=4 min=14 min_rank=2 imbalance_ratio=3.059357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[98106, 24518, 8861, 32079, 3, 27595, 56098, 14884] avg=32768.0000 max=98106 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.993958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[10932, 98, 14, 9479, 100249, 61466, 61319, 18587] avg=32768.0000 max=100249 max_rank=4 min=14 min_rank=2 imbalance_ratio=3.059357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] avg=32768.0000 max=33097 max_rank=3 min=31122 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67152 max_load_reduction_pct=0.669852 moved_to_replicas=123746 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[10932, 98, 14, 9479, 100249, 61466, 61319, 18587] avg=32768.0000 max=100249 max_rank=4 min=14 min_rank=2 imbalance_ratio=3.059357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[10932, 98, 14, 9479, 100249, 61466, 61319, 18587] avg=32768.0000 max=100249 max_rank=4 min=14 min_rank=2 imbalance_ratio=3.059357 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] avg=32768.0000 max=33097 max_rank=3 min=31122 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67152 max_load_reduction_pct=0.669852 moved_to_replicas=123746 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] avg=32768.0000 max=33098 max_rank=1 min=31815 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65008 max_load_reduction_pct=0.662630 moved_to_replicas=88012 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[98106, 24518, 8861, 32079, 3, 27595, 56098, 14884] avg=32768.0000 max=98106 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.993958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] avg=32768.0000 max=33097 max_rank=3 min=31122 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67152 max_load_reduction_pct=0.669852 moved_to_replicas=123746 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[98106, 24518, 8861, 32079, 3, 27595, 56098, 14884] avg=32768.0000 max=98106 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.993958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] avg=32768.0000 max=33097 max_rank=3 min=31122 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67152 max_load_reduction_pct=0.669852 moved_to_replicas=123746 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[33096, 32762, 32781, 33097, 33096, 33096, 33094, 31122] avg=32768.0000 max=33097 max_rank=3 min=31122 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=67152 max_load_reduction_pct=0.669852 moved_to_replicas=123746 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] avg=32768.0000 max=33098 max_rank=1 min=31815 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65008 max_load_reduction_pct=0.662630 moved_to_replicas=88012 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[98106, 24518, 8861, 32079, 3, 27595, 56098, 14884] avg=32768.0000 max=98106 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.993958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] avg=32768.0000 max=33098 max_rank=1 min=31815 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65008 max_load_reduction_pct=0.662630 moved_to_replicas=88012 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[54631, 17988, 9913, 32805, 5, 61609, 72986, 12207] avg=32768.0000 max=72986 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.227356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[98106, 24518, 8861, 32079, 3, 27595, 56098, 14884] avg=32768.0000 max=98106 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.993958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[98106, 24518, 8861, 32079, 3, 27595, 56098, 14884] avg=32768.0000 max=98106 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.993958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] avg=32768.0000 max=33098 max_rank=1 min=31815 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65008 max_load_reduction_pct=0.662630 moved_to_replicas=88012 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[54631, 17988, 9913, 32805, 5, 61609, 72986, 12207] avg=32768.0000 max=72986 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.227356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[98106, 24518, 8861, 32079, 3, 27595, 56098, 14884] avg=32768.0000 max=98106 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.993958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[98106, 24518, 8861, 32079, 3, 27595, 56098, 14884] avg=32768.0000 max=98106 max_rank=0 min=3 min_rank=4 imbalance_ratio=2.993958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] avg=32768.0000 max=33097 max_rank=5 min=31089 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39889 max_load_reduction_pct=0.546529 moved_to_replicas=89938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] avg=32768.0000 max=33098 max_rank=1 min=31815 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65008 max_load_reduction_pct=0.662630 moved_to_replicas=88012 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] avg=32768.0000 max=33098 max_rank=1 min=31815 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65008 max_load_reduction_pct=0.662630 moved_to_replicas=88012 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[54631, 17988, 9913, 32805, 5, 61609, 72986, 12207] avg=32768.0000 max=72986 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.227356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] avg=32768.0000 max=33097 max_rank=5 min=31089 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39889 max_load_reduction_pct=0.546529 moved_to_replicas=89938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] avg=32768.0000 max=33098 max_rank=1 min=31815 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65008 max_load_reduction_pct=0.662630 moved_to_replicas=88012 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[33096, 33098, 33095, 32079, 32769, 31815, 33094, 33098] avg=32768.0000 max=33098 max_rank=1 min=31815 min_rank=5 imbalance_ratio=1.010071 max_load_reduction=65008 max_load_reduction_pct=0.662630 moved_to_replicas=88012 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[54631, 17988, 9913, 32805, 5, 61609, 72986, 12207] avg=32768.0000 max=72986 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.227356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] avg=32768.0000 max=33097 max_rank=5 min=31089 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39889 max_load_reduction_pct=0.546529 moved_to_replicas=89938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[1472, 50223, 5614, 90211, 41927, 65055, 20, 7622] avg=32768.0000 max=90211 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.753021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[54631, 17988, 9913, 32805, 5, 61609, 72986, 12207] avg=32768.0000 max=72986 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.227356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[54631, 17988, 9913, 32805, 5, 61609, 72986, 12207] avg=32768.0000 max=72986 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.227356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] avg=32768.0000 max=33097 max_rank=5 min=31089 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39889 max_load_reduction_pct=0.546529 moved_to_replicas=89938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[1472, 50223, 5614, 90211, 41927, 65055, 20, 7622] avg=32768.0000 max=90211 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.753021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57114 max_load_reduction_pct=0.633116 moved_to_replicas=115032 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] avg=32768.0000 max=33097 max_rank=5 min=31089 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39889 max_load_reduction_pct=0.546529 moved_to_replicas=89938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[54631, 17988, 9913, 32805, 5, 61609, 72986, 12207] avg=32768.0000 max=72986 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.227356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] avg=32768.0000 max=33097 max_rank=5 min=31089 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39889 max_load_reduction_pct=0.546529 moved_to_replicas=89938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[54631, 17988, 9913, 32805, 5, 61609, 72986, 12207] avg=32768.0000 max=72986 max_rank=6 min=5 min_rank=4 imbalance_ratio=2.227356 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[1472, 50223, 5614, 90211, 41927, 65055, 20, 7622] avg=32768.0000 max=90211 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.753021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57114 max_load_reduction_pct=0.633116 moved_to_replicas=115032 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] avg=32768.0000 max=33097 max_rank=5 min=31089 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39889 max_load_reduction_pct=0.546529 moved_to_replicas=89938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[1472, 50223, 5614, 90211, 41927, 65055, 20, 7622] avg=32768.0000 max=90211 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.753021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[33096, 33096, 31089, 32805, 32772, 33097, 33096, 33093] avg=32768.0000 max=33097 max_rank=5 min=31089 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=39889 max_load_reduction_pct=0.546529 moved_to_replicas=89938 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57114 max_load_reduction_pct=0.633116 moved_to_replicas=115032 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[336, 32775, 13816, 32246, 23, 98296, 58050, 26602] avg=32768.0000 max=98296 max_rank=5 min=23 min_rank=4 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[1472, 50223, 5614, 90211, 41927, 65055, 20, 7622] avg=32768.0000 max=90211 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.753021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[1472, 50223, 5614, 90211, 41927, 65055, 20, 7622] avg=32768.0000 max=90211 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.753021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[336, 32775, 13816, 32246, 23, 98296, 58050, 26602] avg=32768.0000 max=98296 max_rank=5 min=23 min_rank=4 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57114 max_load_reduction_pct=0.633116 moved_to_replicas=115032 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] avg=32768.0000 max=33096 max_rank=2 min=32246 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=90154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57114 max_load_reduction_pct=0.633116 moved_to_replicas=115032 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57114 max_load_reduction_pct=0.633116 moved_to_replicas=115032 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[1472, 50223, 5614, 90211, 41927, 65055, 20, 7622] avg=32768.0000 max=90211 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.753021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[1472, 50223, 5614, 90211, 41927, 65055, 20, 7622] avg=32768.0000 max=90211 max_rank=3 min=20 min_rank=6 imbalance_ratio=2.753021 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[336, 32775, 13816, 32246, 23, 98296, 58050, 26602] avg=32768.0000 max=98296 max_rank=5 min=23 min_rank=4 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] avg=32768.0000 max=33096 max_rank=2 min=32246 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=90154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[336, 32775, 13816, 32246, 23, 98296, 58050, 26602] avg=32768.0000 max=98296 max_rank=5 min=23 min_rank=4 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57114 max_load_reduction_pct=0.633116 moved_to_replicas=115032 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[33094, 33096, 33094, 33097, 33097, 33096, 32787, 30783] avg=32768.0000 max=33097 max_rank=3 min=30783 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=57114 max_load_reduction_pct=0.633116 moved_to_replicas=115032 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[38974, 13, 32780, 59418, 64254, 32745, 33952, 8] avg=32768.0000 max=64254 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.960876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] avg=32768.0000 max=33096 max_rank=2 min=32246 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=90154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[336, 32775, 13816, 32246, 23, 98296, 58050, 26602] avg=32768.0000 max=98296 max_rank=5 min=23 min_rank=4 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[336, 32775, 13816, 32246, 23, 98296, 58050, 26602] avg=32768.0000 max=98296 max_rank=5 min=23 min_rank=4 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[38974, 13, 32780, 59418, 64254, 32745, 33952, 8] avg=32768.0000 max=64254 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.960876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] avg=32768.0000 max=33096 max_rank=2 min=32246 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=90154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] avg=32768.0000 max=33098 max_rank=3 min=32022 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=31156 max_load_reduction_pct=0.484888 moved_to_replicas=64214 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] avg=32768.0000 max=33096 max_rank=2 min=32246 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=90154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] avg=32768.0000 max=33096 max_rank=2 min=32246 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=90154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[336, 32775, 13816, 32246, 23, 98296, 58050, 26602] avg=32768.0000 max=98296 max_rank=5 min=23 min_rank=4 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[336, 32775, 13816, 32246, 23, 98296, 58050, 26602] avg=32768.0000 max=98296 max_rank=5 min=23 min_rank=4 imbalance_ratio=2.999756 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[38974, 13, 32780, 59418, 64254, 32745, 33952, 8] avg=32768.0000 max=64254 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.960876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] avg=32768.0000 max=33098 max_rank=3 min=32022 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=31156 max_load_reduction_pct=0.484888 moved_to_replicas=64214 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[38974, 13, 32780, 59418, 64254, 32745, 33952, 8] avg=32768.0000 max=64254 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.960876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] avg=32768.0000 max=33096 max_rank=2 min=32246 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=90154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[32769, 32775, 33096, 32246, 32790, 33096, 33095, 32277] avg=32768.0000 max=33096 max_rank=2 min=32246 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65200 max_load_reduction_pct=0.663303 moved_to_replicas=90154 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] avg=32768.0000 max=33098 max_rank=3 min=32022 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=31156 max_load_reduction_pct=0.484888 moved_to_replicas=64214 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[38974, 13, 32780, 59418, 64254, 32745, 33952, 8] avg=32768.0000 max=64254 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.960876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[38974, 13, 32780, 59418, 64254, 32745, 33952, 8] avg=32768.0000 max=64254 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.960876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] avg=32768.0000 max=33098 max_rank=3 min=32022 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=31156 max_load_reduction_pct=0.484888 moved_to_replicas=64214 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] avg=32768.0000 max=33098 max_rank=3 min=32022 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=31156 max_load_reduction_pct=0.484888 moved_to_replicas=64214 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] avg=32768.0000 max=33098 max_rank=3 min=32022 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=31156 max_load_reduction_pct=0.484888 moved_to_replicas=64214 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[38974, 13, 32780, 59418, 64254, 32745, 33952, 8] avg=32768.0000 max=64254 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.960876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[38974, 13, 32780, 59418, 64254, 32745, 33952, 8] avg=32768.0000 max=64254 max_rank=4 min=8 min_rank=7 imbalance_ratio=1.960876 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] avg=32768.0000 max=33098 max_rank=3 min=32022 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=31156 max_load_reduction_pct=0.484888 moved_to_replicas=64214 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[33097, 32210, 32780, 33098, 33096, 32745, 33096, 32022] avg=32768.0000 max=33098 max_rank=3 min=32022 min_rank=7 imbalance_ratio=1.010071 max_load_reduction=31156 max_load_reduction_pct=0.484888 moved_to_replicas=64214 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=10 total_expected_recv=324778 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=19584 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=10 total_expected_recv=327907 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16640 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=10 total_expected_recv=329616 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=14976 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=10 total_expected_recv=327789 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=17664 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=10 total_expected_recv=328396 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16384 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=10 total_expected_recv=327114 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=17536 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=10 total_expected_recv=329421 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16128 peak_demand=33098 required_capacity=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=10 total_expected_recv=326419 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=18816 peak_demand=33098 required_capacity=1.010071 + [2026-08-26 18:02:40] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 634.7 | throughput per GPU (TFLOP/s/GPU): 194.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.637519E+00 | loss scale: 1.0 | grad norm: 4.365 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=34816 expected_recv=31759 expected_overflow=0 exact_recv_per_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=34816 expected_recv=32083 expected_overflow=0 exact_recv_per_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=34816 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=34816 expected_recv=31759 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=34816 expected_recv=32083 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=34816 expected_recv=32822 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=32768 slack_slots=2048 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=34816 expected_recv=31274 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=34816 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=34816 expected_recv=32295 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=34816 expected_recv=31274 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=34816 expected_recv=33094 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=31744 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=34816 expected_recv=32295 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=34816 expected_recv=31547 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=34816 expected_recv=32560 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=34816 expected_recv=31547 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=34816 expected_recv=32560 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=34816 expected_recv=32557 expected_overflow=0 exact_recv_per_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=34816 expected_recv=32557 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=34816 expected_recv=32559 expected_overflow=0 exact_recv_per_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=34816 expected_recv=32559 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=34816 expected_recv=31008 expected_overflow=0 exact_recv_per_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=34816 expected_recv=31008 actual_valid_recv=31616 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=31616 slack_slots=3200 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=34816 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=34816 expected_recv=32838 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=34816 expected_recv=31047 expected_overflow=0 exact_recv_per_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=34816 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=34816 expected_recv=31047 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=34816 expected_recv=32779 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=32000 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=34816 expected_recv=31899 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=34816 expected_recv=31899 actual_valid_recv=32256 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=34816 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=32256 slack_slots=2560 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=34816 expected_recv=32681 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=34816 expected_recv=32406 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33920 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=34816 expected_recv=32406 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=34816 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33920 slack_slots=896 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=34816 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=34816 expected_recv=32774 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=34816 expected_recv=32809 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=34816 expected_recv=31063 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=34816 expected_recv=31063 actual_valid_recv=31488 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=31488 slack_slots=3328 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=34816 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=34816 expected_recv=31078 expected_overflow=0 exact_recv_per_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=34816 expected_recv=32792 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=34816 expected_recv=31078 actual_valid_recv=32000 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=32000 slack_slots=2816 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=34816 expected_recv=32540 expected_overflow=0 exact_recv_per_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=34816 expected_recv=33044 expected_overflow=0 exact_recv_per_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=34816 expected_recv=32540 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=34816 expected_recv=33044 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=34816 expected_recv=31189 expected_overflow=0 exact_recv_per_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=34816 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33792 slack_slots=1024 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=34816 expected_recv=31189 actual_valid_recv=31744 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=34816 expected_recv=32772 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=31744 slack_slots=3072 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=34816 expected_recv=33097 expected_overflow=0 exact_recv_per_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=34816 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=34816 expected_recv=33097 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=34816 expected_recv=33060 expected_overflow=0 exact_recv_per_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=34816 expected_recv=32778 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=34816 expected_recv=32412 expected_overflow=0 exact_recv_per_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=34816 expected_recv=33056 expected_overflow=0 exact_recv_per_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=34816 expected_recv=33060 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=34816 expected_recv=32412 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=34816 expected_recv=33056 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=34816 expected_recv=32522 expected_overflow=0 exact_recv_per_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33024 slack_slots=1792 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=34816 expected_recv=32522 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33664 slack_slots=1152 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=34816 expected_recv=33054 expected_overflow=0 exact_recv_per_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=34816 expected_recv=33095 expected_overflow=0 exact_recv_per_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=34816 expected_recv=33054 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33536 slack_slots=1280 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=34816 expected_recv=33095 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33280 slack_slots=1536 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=34816 expected_recv=31773 expected_overflow=0 exact_recv_per_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[37442, 32822, 27317, 29082, 35304, 28343, 36061, 35773] avg=32768.0000 max=37442 max_rank=0 min=27317 min_rank=2 imbalance_ratio=1.142639 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33408 slack_slots=1408 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=34816 expected_recv=31773 actual_valid_recv=32128 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=34816 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[37442, 32822, 27317, 29082, 35304, 28343, 36061, 35773] avg=32768.0000 max=37442 max_rank=0 min=27317 min_rank=2 imbalance_ratio=1.142639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[37442, 32822, 27317, 29082, 35304, 28343, 36061, 35773] avg=32768.0000 max=37442 max_rank=0 min=27317 min_rank=2 imbalance_ratio=1.142639 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=32128 slack_slots=2688 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=34816 expected_recv=33096 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31759 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=4345 max_load_reduction_pct=0.116046 moved_to_replicas=12196 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=34816 valid_num_tokens=33152 slack_slots=1664 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[37442, 32822, 27317, 29082, 35304, 28343, 36061, 35773] avg=32768.0000 max=37442 max_rank=0 min=27317 min_rank=2 imbalance_ratio=1.142639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[37442, 32822, 27317, 29082, 35304, 28343, 36061, 35773] avg=32768.0000 max=37442 max_rank=0 min=27317 min_rank=2 imbalance_ratio=1.142639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31759 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=4345 max_load_reduction_pct=0.116046 moved_to_replicas=12196 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[37442, 32822, 27317, 29082, 35304, 28343, 36061, 35773] avg=32768.0000 max=37442 max_rank=0 min=27317 min_rank=2 imbalance_ratio=1.142639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31759 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=4345 max_load_reduction_pct=0.116046 moved_to_replicas=12196 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[37442, 32822, 27317, 29082, 35304, 28343, 36061, 35773] avg=32768.0000 max=37442 max_rank=0 min=27317 min_rank=2 imbalance_ratio=1.142639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31759 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=4345 max_load_reduction_pct=0.116046 moved_to_replicas=12196 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[7425, 55555, 9723, 8309, 9836, 28068, 55296, 87932] avg=32768.0000 max=87932 max_rank=7 min=7425 min_rank=0 imbalance_ratio=2.683472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31759 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=4345 max_load_reduction_pct=0.116046 moved_to_replicas=12196 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31759 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=4345 max_load_reduction_pct=0.116046 moved_to_replicas=12196 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[37442, 32822, 27317, 29082, 35304, 28343, 36061, 35773] avg=32768.0000 max=37442 max_rank=0 min=27317 min_rank=2 imbalance_ratio=1.142639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[7425, 55555, 9723, 8309, 9836, 28068, 55296, 87932] avg=32768.0000 max=87932 max_rank=7 min=7425 min_rank=0 imbalance_ratio=2.683472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[7425, 55555, 9723, 8309, 9836, 28068, 55296, 87932] avg=32768.0000 max=87932 max_rank=7 min=7425 min_rank=0 imbalance_ratio=2.683472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31759 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=4345 max_load_reduction_pct=0.116046 moved_to_replicas=12196 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31274 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54835 max_load_reduction_pct=0.623607 moved_to_replicas=99495 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31274 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54835 max_load_reduction_pct=0.623607 moved_to_replicas=99495 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[33096, 32822, 32083, 31759, 33096, 33097, 33095, 33096] avg=32768.0000 max=33097 max_rank=5 min=31759 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=4345 max_load_reduction_pct=0.116046 moved_to_replicas=12196 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[7425, 55555, 9723, 8309, 9836, 28068, 55296, 87932] avg=32768.0000 max=87932 max_rank=7 min=7425 min_rank=0 imbalance_ratio=2.683472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[7425, 55555, 9723, 8309, 9836, 28068, 55296, 87932] avg=32768.0000 max=87932 max_rank=7 min=7425 min_rank=0 imbalance_ratio=2.683472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[7425, 55555, 9723, 8309, 9836, 28068, 55296, 87932] avg=32768.0000 max=87932 max_rank=7 min=7425 min_rank=0 imbalance_ratio=2.683472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31274 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54835 max_load_reduction_pct=0.623607 moved_to_replicas=99495 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[7425, 55555, 9723, 8309, 9836, 28068, 55296, 87932] avg=32768.0000 max=87932 max_rank=7 min=7425 min_rank=0 imbalance_ratio=2.683472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32557, 2063, 32560, 2789, 80881, 30070, 3675, 77549] avg=32768.0000 max=80881 max_rank=4 min=2063 min_rank=1 imbalance_ratio=2.468292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31274 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54835 max_load_reduction_pct=0.623607 moved_to_replicas=99495 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31274 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54835 max_load_reduction_pct=0.623607 moved_to_replicas=99495 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[32557, 2063, 32560, 2789, 80881, 30070, 3675, 77549] avg=32768.0000 max=80881 max_rank=4 min=2063 min_rank=1 imbalance_ratio=2.468292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31274 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54835 max_load_reduction_pct=0.623607 moved_to_replicas=99495 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[7425, 55555, 9723, 8309, 9836, 28068, 55296, 87932] avg=32768.0000 max=87932 max_rank=7 min=7425 min_rank=0 imbalance_ratio=2.683472 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31274 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54835 max_load_reduction_pct=0.623607 moved_to_replicas=99495 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32557, 2063, 32560, 2789, 80881, 30070, 3675, 77549] avg=32768.0000 max=80881 max_rank=4 min=2063 min_rank=1 imbalance_ratio=2.468292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31547 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47784 max_load_reduction_pct=0.590794 moved_to_replicas=92238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31547 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47784 max_load_reduction_pct=0.590794 moved_to_replicas=92238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[32557, 2063, 32560, 2789, 80881, 30070, 3675, 77549] avg=32768.0000 max=80881 max_rank=4 min=2063 min_rank=1 imbalance_ratio=2.468292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[33096, 33096, 33094, 33096, 32295, 31274, 33096, 33097] avg=32768.0000 max=33097 max_rank=7 min=31274 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=54835 max_load_reduction_pct=0.623607 moved_to_replicas=99495 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31547 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47784 max_load_reduction_pct=0.590794 moved_to_replicas=92238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[32557, 2063, 32560, 2789, 80881, 30070, 3675, 77549] avg=32768.0000 max=80881 max_rank=4 min=2063 min_rank=1 imbalance_ratio=2.468292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32557, 2063, 32560, 2789, 80881, 30070, 3675, 77549] avg=32768.0000 max=80881 max_rank=4 min=2063 min_rank=1 imbalance_ratio=2.468292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32557, 2063, 32560, 2789, 80881, 30070, 3675, 77549] avg=32768.0000 max=80881 max_rank=4 min=2063 min_rank=1 imbalance_ratio=2.468292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[26956, 5520, 86, 33356, 57919, 48342, 57406, 32559] avg=32768.0000 max=57919 max_rank=4 min=86 min_rank=2 imbalance_ratio=1.767548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31547 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47784 max_load_reduction_pct=0.590794 moved_to_replicas=92238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31547 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47784 max_load_reduction_pct=0.590794 moved_to_replicas=92238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[26956, 5520, 86, 33356, 57919, 48342, 57406, 32559] avg=32768.0000 max=57919 max_rank=4 min=86 min_rank=2 imbalance_ratio=1.767548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31547 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47784 max_load_reduction_pct=0.590794 moved_to_replicas=92238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32557, 2063, 32560, 2789, 80881, 30070, 3675, 77549] avg=32768.0000 max=80881 max_rank=4 min=2063 min_rank=1 imbalance_ratio=2.468292 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31547 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47784 max_load_reduction_pct=0.590794 moved_to_replicas=92238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] avg=32768.0000 max=33097 max_rank=0 min=31008 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=24822 max_load_reduction_pct=0.428564 moved_to_replicas=64639 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[26956, 5520, 86, 33356, 57919, 48342, 57406, 32559] avg=32768.0000 max=57919 max_rank=4 min=86 min_rank=2 imbalance_ratio=1.767548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] avg=32768.0000 max=33097 max_rank=0 min=31008 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=24822 max_load_reduction_pct=0.428564 moved_to_replicas=64639 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[26956, 5520, 86, 33356, 57919, 48342, 57406, 32559] avg=32768.0000 max=57919 max_rank=4 min=86 min_rank=2 imbalance_ratio=1.767548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32557, 33096, 32560, 33095, 33097, 31547, 33096, 33096] avg=32768.0000 max=33097 max_rank=4 min=31547 min_rank=5 imbalance_ratio=1.010040 max_load_reduction=47784 max_load_reduction_pct=0.590794 moved_to_replicas=92238 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[26956, 5520, 86, 33356, 57919, 48342, 57406, 32559] avg=32768.0000 max=57919 max_rank=4 min=86 min_rank=2 imbalance_ratio=1.767548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[26956, 5520, 86, 33356, 57919, 48342, 57406, 32559] avg=32768.0000 max=57919 max_rank=4 min=86 min_rank=2 imbalance_ratio=1.767548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] avg=32768.0000 max=33097 max_rank=0 min=31008 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=24822 max_load_reduction_pct=0.428564 moved_to_replicas=64639 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[18585, 17, 5566, 1403, 65550, 21677, 113813, 35533] avg=32768.0000 max=113813 max_rank=6 min=17 min_rank=1 imbalance_ratio=3.473297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[26956, 5520, 86, 33356, 57919, 48342, 57406, 32559] avg=32768.0000 max=57919 max_rank=4 min=86 min_rank=2 imbalance_ratio=1.767548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] avg=32768.0000 max=33097 max_rank=0 min=31008 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=24822 max_load_reduction_pct=0.428564 moved_to_replicas=64639 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] avg=32768.0000 max=33097 max_rank=0 min=31008 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=24822 max_load_reduction_pct=0.428564 moved_to_replicas=64639 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[18585, 17, 5566, 1403, 65550, 21677, 113813, 35533] avg=32768.0000 max=113813 max_rank=6 min=17 min_rank=1 imbalance_ratio=3.473297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] avg=32768.0000 max=33097 max_rank=0 min=31008 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=24822 max_load_reduction_pct=0.428564 moved_to_replicas=64639 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[26956, 5520, 86, 33356, 57919, 48342, 57406, 32559] avg=32768.0000 max=57919 max_rank=4 min=86 min_rank=2 imbalance_ratio=1.767548 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80717 max_load_reduction_pct=0.709207 moved_to_replicas=115608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] avg=32768.0000 max=33097 max_rank=0 min=31008 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=24822 max_load_reduction_pct=0.428564 moved_to_replicas=64639 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[18585, 17, 5566, 1403, 65550, 21677, 113813, 35533] avg=32768.0000 max=113813 max_rank=6 min=17 min_rank=1 imbalance_ratio=3.473297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80717 max_load_reduction_pct=0.709207 moved_to_replicas=115608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[18585, 17, 5566, 1403, 65550, 21677, 113813, 35533] avg=32768.0000 max=113813 max_rank=6 min=17 min_rank=1 imbalance_ratio=3.473297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[33097, 31008, 33095, 33096, 33096, 33096, 33097, 32559] avg=32768.0000 max=33097 max_rank=0 min=31008 min_rank=1 imbalance_ratio=1.010040 max_load_reduction=24822 max_load_reduction_pct=0.428564 moved_to_replicas=64639 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[18585, 17, 5566, 1403, 65550, 21677, 113813, 35533] avg=32768.0000 max=113813 max_rank=6 min=17 min_rank=1 imbalance_ratio=3.473297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[18585, 17, 5566, 1403, 65550, 21677, 113813, 35533] avg=32768.0000 max=113813 max_rank=6 min=17 min_rank=1 imbalance_ratio=3.473297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80717 max_load_reduction_pct=0.709207 moved_to_replicas=115608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[98242, 21, 33299, 828, 13, 33786, 63274, 32681] avg=32768.0000 max=98242 max_rank=0 min=13 min_rank=4 imbalance_ratio=2.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80717 max_load_reduction_pct=0.709207 moved_to_replicas=115608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[18585, 17, 5566, 1403, 65550, 21677, 113813, 35533] avg=32768.0000 max=113813 max_rank=6 min=17 min_rank=1 imbalance_ratio=3.473297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[98242, 21, 33299, 828, 13, 33786, 63274, 32681] avg=32768.0000 max=98242 max_rank=0 min=13 min_rank=4 imbalance_ratio=2.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80717 max_load_reduction_pct=0.709207 moved_to_replicas=115608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80717 max_load_reduction_pct=0.709207 moved_to_replicas=115608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[18585, 17, 5566, 1403, 65550, 21677, 113813, 35533] avg=32768.0000 max=113813 max_rank=6 min=17 min_rank=1 imbalance_ratio=3.473297 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65146 max_load_reduction_pct=0.663118 moved_to_replicas=96217 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80717 max_load_reduction_pct=0.709207 moved_to_replicas=115608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[98242, 21, 33299, 828, 13, 33786, 63274, 32681] avg=32768.0000 max=98242 max_rank=0 min=13 min_rank=4 imbalance_ratio=2.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65146 max_load_reduction_pct=0.663118 moved_to_replicas=96217 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[98242, 21, 33299, 828, 13, 33786, 63274, 32681] avg=32768.0000 max=98242 max_rank=0 min=13 min_rank=4 imbalance_ratio=2.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[33096, 32779, 31047, 32838, 33096, 33096, 33096, 33096] avg=32768.0000 max=33096 max_rank=0 min=31047 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=80717 max_load_reduction_pct=0.709207 moved_to_replicas=115608 active_clone_links=7 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[98242, 21, 33299, 828, 13, 33786, 63274, 32681] avg=32768.0000 max=98242 max_rank=0 min=13 min_rank=4 imbalance_ratio=2.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[98242, 21, 33299, 828, 13, 33786, 63274, 32681] avg=32768.0000 max=98242 max_rank=0 min=13 min_rank=4 imbalance_ratio=2.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[24791, 17981, 31058, 58849, 30, 32809, 96503, 123] avg=32768.0000 max=96503 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.945038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65146 max_load_reduction_pct=0.663118 moved_to_replicas=96217 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[98242, 21, 33299, 828, 13, 33786, 63274, 32681] avg=32768.0000 max=98242 max_rank=0 min=13 min_rank=4 imbalance_ratio=2.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65146 max_load_reduction_pct=0.663118 moved_to_replicas=96217 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[24791, 17981, 31058, 58849, 30, 32809, 96503, 123] avg=32768.0000 max=96503 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.945038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65146 max_load_reduction_pct=0.663118 moved_to_replicas=96217 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65146 max_load_reduction_pct=0.663118 moved_to_replicas=96217 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31063 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=63406 max_load_reduction_pct=0.657037 moved_to_replicas=89160 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[98242, 21, 33299, 828, 13, 33786, 63274, 32681] avg=32768.0000 max=98242 max_rank=0 min=13 min_rank=4 imbalance_ratio=2.998108 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65146 max_load_reduction_pct=0.663118 moved_to_replicas=96217 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31063 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=63406 max_load_reduction_pct=0.657037 moved_to_replicas=89160 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[24791, 17981, 31058, 58849, 30, 32809, 96503, 123] avg=32768.0000 max=96503 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.945038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[24791, 17981, 31058, 58849, 30, 32809, 96503, 123] avg=32768.0000 max=96503 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.945038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[33096, 32406, 33096, 31899, 32774, 33096, 33096, 32681] avg=32768.0000 max=33096 max_rank=0 min=31899 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=65146 max_load_reduction_pct=0.663118 moved_to_replicas=96217 active_clone_links=5 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[24791, 17981, 31058, 58849, 30, 32809, 96503, 123] avg=32768.0000 max=96503 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.945038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[24791, 17981, 31058, 58849, 30, 32809, 96503, 123] avg=32768.0000 max=96503 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.945038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[278, 33660, 32540, 67666, 65394, 45750, 9116, 7740] avg=32768.0000 max=67666 max_rank=3 min=278 min_rank=0 imbalance_ratio=2.065002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31063 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=63406 max_load_reduction_pct=0.657037 moved_to_replicas=89160 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[24791, 17981, 31058, 58849, 30, 32809, 96503, 123] avg=32768.0000 max=96503 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.945038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[278, 33660, 32540, 67666, 65394, 45750, 9116, 7740] avg=32768.0000 max=67666 max_rank=3 min=278 min_rank=0 imbalance_ratio=2.065002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31063 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=63406 max_load_reduction_pct=0.657037 moved_to_replicas=89160 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31063 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=63406 max_load_reduction_pct=0.657037 moved_to_replicas=89160 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31063 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=63406 max_load_reduction_pct=0.657037 moved_to_replicas=89160 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] avg=32768.0000 max=33097 max_rank=5 min=31078 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34569 max_load_reduction_pct=0.510877 moved_to_replicas=80086 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[24791, 17981, 31058, 58849, 30, 32809, 96503, 123] avg=32768.0000 max=96503 max_rank=6 min=30 min_rank=4 imbalance_ratio=2.945038 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31063 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=63406 max_load_reduction_pct=0.657037 moved_to_replicas=89160 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] avg=32768.0000 max=33097 max_rank=5 min=31078 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34569 max_load_reduction_pct=0.510877 moved_to_replicas=80086 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[278, 33660, 32540, 67666, 65394, 45750, 9116, 7740] avg=32768.0000 max=67666 max_rank=3 min=278 min_rank=0 imbalance_ratio=2.065002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[278, 33660, 32540, 67666, 65394, 45750, 9116, 7740] avg=32768.0000 max=67666 max_rank=3 min=278 min_rank=0 imbalance_ratio=2.065002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[33095, 33097, 31063, 33096, 32792, 32809, 33096, 33096] avg=32768.0000 max=33097 max_rank=1 min=31063 min_rank=2 imbalance_ratio=1.010040 max_load_reduction=63406 max_load_reduction_pct=0.657037 moved_to_replicas=89160 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[278, 33660, 32540, 67666, 65394, 45750, 9116, 7740] avg=32768.0000 max=67666 max_rank=3 min=278 min_rank=0 imbalance_ratio=2.065002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[278, 33660, 32540, 67666, 65394, 45750, 9116, 7740] avg=32768.0000 max=67666 max_rank=3 min=278 min_rank=0 imbalance_ratio=2.065002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[33060, 24004, 16, 31189, 737, 98265, 42101, 32772] avg=32768.0000 max=98265 max_rank=5 min=16 min_rank=2 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] avg=32768.0000 max=33097 max_rank=5 min=31078 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34569 max_load_reduction_pct=0.510877 moved_to_replicas=80086 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[33060, 24004, 16, 31189, 737, 98265, 42101, 32772] avg=32768.0000 max=98265 max_rank=5 min=16 min_rank=2 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[278, 33660, 32540, 67666, 65394, 45750, 9116, 7740] avg=32768.0000 max=67666 max_rank=3 min=278 min_rank=0 imbalance_ratio=2.065002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] avg=32768.0000 max=33097 max_rank=5 min=31078 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34569 max_load_reduction_pct=0.510877 moved_to_replicas=80086 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] avg=32768.0000 max=33097 max_rank=5 min=31078 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34569 max_load_reduction_pct=0.510877 moved_to_replicas=80086 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] avg=32768.0000 max=33097 max_rank=6 min=31189 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65168 max_load_reduction_pct=0.663186 moved_to_replicas=74174 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] avg=32768.0000 max=33097 max_rank=5 min=31078 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34569 max_load_reduction_pct=0.510877 moved_to_replicas=80086 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[278, 33660, 32540, 67666, 65394, 45750, 9116, 7740] avg=32768.0000 max=67666 max_rank=3 min=278 min_rank=0 imbalance_ratio=2.065002 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] avg=32768.0000 max=33097 max_rank=6 min=31189 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65168 max_load_reduction_pct=0.663186 moved_to_replicas=74174 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] avg=32768.0000 max=33097 max_rank=5 min=31078 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34569 max_load_reduction_pct=0.510877 moved_to_replicas=80086 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[33060, 24004, 16, 31189, 737, 98265, 42101, 32772] avg=32768.0000 max=98265 max_rank=5 min=16 min_rank=2 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[33044, 33096, 32540, 33096, 33096, 33097, 33097, 31078] avg=32768.0000 max=33097 max_rank=5 min=31078 min_rank=7 imbalance_ratio=1.010040 max_load_reduction=34569 max_load_reduction_pct=0.510877 moved_to_replicas=80086 active_clone_links=6 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[33060, 24004, 16, 31189, 737, 98265, 42101, 32772] avg=32768.0000 max=98265 max_rank=5 min=16 min_rank=2 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[33060, 24004, 16, 31189, 737, 98265, 42101, 32772] avg=32768.0000 max=98265 max_rank=5 min=16 min_rank=2 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[53374, 1202, 33054, 63667, 41804, 63, 65445, 3535] avg=32768.0000 max=65445 max_rank=6 min=63 min_rank=5 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[33060, 24004, 16, 31189, 737, 98265, 42101, 32772] avg=32768.0000 max=98265 max_rank=5 min=16 min_rank=2 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] avg=32768.0000 max=33097 max_rank=6 min=31189 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65168 max_load_reduction_pct=0.663186 moved_to_replicas=74174 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[53374, 1202, 33054, 63667, 41804, 63, 65445, 3535] avg=32768.0000 max=65445 max_rank=6 min=63 min_rank=5 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[33060, 24004, 16, 31189, 737, 98265, 42101, 32772] avg=32768.0000 max=98265 max_rank=5 min=16 min_rank=2 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] avg=32768.0000 max=33097 max_rank=6 min=31189 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65168 max_load_reduction_pct=0.663186 moved_to_replicas=74174 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] avg=32768.0000 max=33097 max_rank=6 min=31189 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65168 max_load_reduction_pct=0.663186 moved_to_replicas=74174 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] avg=32768.0000 max=33096 max_rank=3 min=31773 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32349 max_load_reduction_pct=0.494293 moved_to_replicas=91906 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] avg=32768.0000 max=33097 max_rank=6 min=31189 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65168 max_load_reduction_pct=0.663186 moved_to_replicas=74174 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[33060, 24004, 16, 31189, 737, 98265, 42101, 32772] avg=32768.0000 max=98265 max_rank=5 min=16 min_rank=2 imbalance_ratio=2.998810 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] avg=32768.0000 max=33096 max_rank=3 min=31773 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32349 max_load_reduction_pct=0.494293 moved_to_replicas=91906 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] avg=32768.0000 max=33097 max_rank=6 min=31189 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65168 max_load_reduction_pct=0.663186 moved_to_replicas=74174 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[53374, 1202, 33054, 63667, 41804, 63, 65445, 3535] avg=32768.0000 max=65445 max_rank=6 min=63 min_rank=5 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[33060, 33056, 32778, 31189, 33096, 33096, 33097, 32772] avg=32768.0000 max=33097 max_rank=6 min=31189 min_rank=3 imbalance_ratio=1.010040 max_load_reduction=65168 max_load_reduction_pct=0.663186 moved_to_replicas=74174 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[53374, 1202, 33054, 63667, 41804, 63, 65445, 3535] avg=32768.0000 max=65445 max_rank=6 min=63 min_rank=5 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[53374, 1202, 33054, 63667, 41804, 63, 65445, 3535] avg=32768.0000 max=65445 max_rank=6 min=63 min_rank=5 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[53374, 1202, 33054, 63667, 41804, 63, 65445, 3535] avg=32768.0000 max=65445 max_rank=6 min=63 min_rank=5 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] avg=32768.0000 max=33096 max_rank=3 min=31773 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32349 max_load_reduction_pct=0.494293 moved_to_replicas=91906 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[53374, 1202, 33054, 63667, 41804, 63, 65445, 3535] avg=32768.0000 max=65445 max_rank=6 min=63 min_rank=5 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] avg=32768.0000 max=33096 max_rank=3 min=31773 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32349 max_load_reduction_pct=0.494293 moved_to_replicas=91906 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] avg=32768.0000 max=33096 max_rank=3 min=31773 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32349 max_load_reduction_pct=0.494293 moved_to_replicas=91906 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] avg=32768.0000 max=33096 max_rank=3 min=31773 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32349 max_load_reduction_pct=0.494293 moved_to_replicas=91906 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[53374, 1202, 33054, 63667, 41804, 63, 65445, 3535] avg=32768.0000 max=65445 max_rank=6 min=63 min_rank=5 imbalance_ratio=1.997223 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] avg=32768.0000 max=33096 max_rank=3 min=31773 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32349 max_load_reduction_pct=0.494293 moved_to_replicas=91906 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[33095, 31773, 33054, 33096, 33096, 32412, 33096, 32522] avg=32768.0000 max=33096 max_rank=3 min=31773 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=32349 max_load_reduction_pct=0.494293 moved_to_replicas=91906 active_clone_links=4 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=10 total_expected_recv=326620 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16640 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=10 total_expected_recv=327093 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=15616 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=10 total_expected_recv=326260 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=17152 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=10 total_expected_recv=330332 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=13824 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=10 total_expected_recv=330962 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=12160 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=10 total_expected_recv=324410 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=18176 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=10 total_expected_recv=326229 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=16640 peak_demand=33097 required_capacity=1.010040 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=10 total_expected_recv=329534 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=14080 peak_demand=33097 required_capacity=1.010040 + [2026-08-26 18:02:40] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 240.7 | throughput per GPU (TFLOP/s/GPU): 513.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.561920E+00 | loss scale: 1.0 | grad norm: 4.307 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 34048 | min safe capacity: 1.0391 | +[after training is done] datetime: 2026-08-26 18:02:40 +[rank0]:[W826 18:02:41.001241421 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 18:02:42.905000 446338 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 18:02:42.906000 446338 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 446428 closing signal SIGTERM +W0826 18:02:42.906000 446338 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 446429 closing signal SIGTERM +W0826 18:02:42.909000 446338 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 446430 closing signal SIGTERM +W0826 18:02:42.911000 446338 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 446431 closing signal SIGTERM +W0826 18:02:42.915000 446338 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 446432 closing signal SIGTERM +W0826 18:02:42.919000 446338 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 446433 closing signal SIGTERM +W0826 18:02:42.921000 446338 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 446434 closing signal SIGTERM +W0826 18:02:42.922000 446338 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 446435 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 446338 got signal: 15 diff --git a/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv b/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv index 1f7138078d06644166969774683362fa9c445ab9..291a203ff7c2a6ae0df73ac350c09921e6bfbf03 100644 --- a/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv +++ b/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/mem_samples.csv @@ -14,7 +14,7 @@ 5, 0 6, 0 7, 0 -0, 1 +0, 4 1, 1 2, 1 3, 1 @@ -38,54 +38,54 @@ 5, 4 6, 4 7, 4 -0, 4 -1, 27 -2, 27 -3, 47 -4, 27 -5, 27 -6, 27 -7, 27 -0, 891 -1, 891 -2, 921 -3, 915 -4, 903 -5, 907 -6, 891 -7, 917 -0, 1803 -1, 1803 -2, 1803 -3, 1803 -4, 1803 -5, 1803 -6, 1803 -7, 1803 -0, 2441 -1, 2441 -2, 2441 -3, 2441 -4, 2441 -5, 2435 -6, 2431 -7, 2441 -0, 46703 -1, 46137 -2, 46137 -3, 46837 -4, 46837 -5, 46837 -6, 46837 -7, 46837 -0, 47109 -1, 46539 -2, 46539 -3, 47239 -4, 47241 -5, 47239 -6, 47239 -7, 47239 +0, 821 +1, 821 +2, 819 +3, 821 +4, 821 +5, 819 +6, 821 +7, 821 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 2831 +1, 2831 +2, 2831 +3, 2831 +4, 2831 +5, 2831 +6, 2831 +7, 2831 +0, 46953 +1, 46379 +2, 46341 +3, 47043 +4, 47037 +5, 47057 +6, 47039 +7, 47073 +0, 47833 +1, 47303 +2, 47293 +3, 47983 +4, 48003 +5, 48013 +6, 48013 +7, 48033 0, 48395 1, 47825 2, 47825 @@ -102,14 +102,6 @@ 5, 48805 6, 48805 7, 48805 -0, 48675 -1, 48105 -2, 48105 -3, 48805 -4, 48805 -5, 48805 -6, 48805 -7, 48805 0, 50879 1, 50309 2, 50309 @@ -118,14 +110,22 @@ 5, 51009 6, 51009 7, 51009 -0, 74423 -1, 74481 -2, 74435 -3, 74481 -4, 74473 -5, 74469 -6, 74491 -7, 74471 +0, 56865 +1, 57181 +2, 56799 +3, 57195 +4, 56809 +5, 56871 +6, 57191 +7, 57193 +0, 74975 +1, 75011 +2, 74991 +3, 75021 +4, 75021 +5, 75021 +6, 75021 +7, 75021 0, 74975 1, 75011 2, 74991 @@ -142,22 +142,22 @@ 5, 75021 6, 75021 7, 75021 -0, 75203 -1, 75335 -2, 75237 -3, 75297 -4, 75253 -5, 75309 -6, 75283 -7, 75245 -0, 81777 -1, 81671 -2, 81675 -3, 81717 -4, 81721 -5, 81713 -6, 81691 -7, 81695 +0, 75291 +1, 75345 +2, 75239 +3, 75321 +4, 75331 +5, 75307 +6, 75341 +7, 75343 +0, 81867 +1, 81723 +2, 81667 +3, 81723 +4, 81717 +5, 81701 +6, 81723 +7, 81733 0, 97625 1, 97677 2, 97675 @@ -182,10 +182,10 @@ 5, 96657 6, 96657 7, 96657 -0, 257 -1, 93831 -2, 1284 -3, 4 +0, 1 +1, 260 +2, 93829 +3, 1284 4, 4 5, 4 6, 4 diff --git a/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log b/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log index c0a4984d580d33798f64b5c7408172c25328068b..63fc07dd26ac719aeb44ccaa644c4f5ebc195c8f 100644 --- a/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log +++ b/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/train.log @@ -1,4 +1,4 @@ -[16:50:41] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[18:02:48] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) ============================================================== QWEN2_57B / ultraep (slots/rank=8 policy=matched) E=64 K=8 H'=2560 m=8 @@ -7,10 +7,22 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-UltraEP @ eaf829e7 ============================================================== -W0826 16:50:42.495000 276802 torch/distributed/run.py:766] -W0826 16:50:42.495000 276802 torch/distributed/run.py:766] ***************************************** -W0826 16:50:42.495000 276802 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 16:50:42.495000 276802 torch/distributed/run.py:766] ***************************************** +W0826 18:02:49.604000 449269 torch/distributed/run.py:766] +W0826 18:02:49.604000 449269 torch/distributed/run.py:766] ***************************************** +W0826 18:02:49.604000 449269 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 18:02:49.604000 449269 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer Number of virtual stages per pipeline stage: None @@ -565,40 +577,28 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... - > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) -WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS -> initializing torch distributed ... -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it /workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-UltraEP/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 6 is connected to 77 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to [Gloo] Rank 7[Gloo] Rank peer ranks. 6Expected number of connected peer ranks is : 5 is connected to 7 is connected to 7 -7 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 - -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 @@ -607,27 +607,27 @@ WARNING: one_logger package is required to enable e2e metrics tracking. please g [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank1]:[W826 16:51:00.990483205 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 16:51:00.993899871 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 16:51:00.994036963 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 16:51:00.994076971 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 16:51:00.994087523 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 16:51:00.994167192 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 16:51:00.994243536 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 18:03:08.117096648 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 18:03:08.117192966 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 18:03:08.121931861 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 18:03:08.121949954 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 18:03:08.121955352 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 18:03:08.122031750 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 18:03:08.123472003 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-UltraEP/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-UltraEP/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.035 seconds +>>> done with dataset index builder. Compilation time: 0.034 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 16:51:01.142750705 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.629 seconds -time to initialize megatron (seconds): 6.928 -[after megatron is initialized] datetime: 2026-08-26 16:51:06 +[rank0]:[W826 18:03:08.271326330 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.677 seconds +time to initialize megatron (seconds): 6.768 +[after megatron is initialized] datetime: 2026-08-26 18:03:13 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -676,114 +676,114 @@ Params for bucket 2 (73403904 elements, 73403904 padded size): Params for bucket 3 (146800640 elements, 146800640 padded size): module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight Params for bucket 4 (103001600 elements, 103001600 padded size): + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.mlp.router.weight module.decoder.layers.9.self_attention.linear_qkv.bias module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.9.self_attention.linear_qkv.weight - module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight Params for bucket 5 (146800640 elements, 146800640 padded size): module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight Params for bucket 6 (103001600 elements, 103001600 padded size): + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.mlp.router.weight module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.linear_qkv.bias module.decoder.layers.8.self_attention.linear_proj.weight module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.8.self_attention.linear_qkv.weight - module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight Params for bucket 7 (146800640 elements, 146800640 padded size): module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight Params for bucket 8 (103001600 elements, 103001600 padded size): + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.7.mlp.router.weight module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.self_attention.linear_qkv.bias module.decoder.layers.7.self_attention.linear_proj.weight module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.7.self_attention.linear_qkv.weight Params for bucket 9 (146800640 elements, 146800640 padded size): module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight Params for bucket 10 (103001600 elements, 103001600 padded size): - module.decoder.layers.6.mlp.router.weight - module.decoder.layers.6.pre_mlp_layernorm.weight - module.decoder.layers.6.self_attention.linear_qkv.weight - module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.6.self_attention.linear_qkv.bias + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.pre_mlp_layernorm.weight module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight Params for bucket 11 (146800640 elements, 146800640 padded size): module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight Params for bucket 12 (103001600 elements, 103001600 padded size): - module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight module.decoder.layers.5.mlp.router.weight module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.5.self_attention.linear_qkv.bias module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight Params for bucket 13 (146800640 elements, 146800640 padded size): module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight Params for bucket 14 (103001600 elements, 103001600 padded size): + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_qkv.weight module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.linear_qkv.bias module.decoder.layers.4.self_attention.linear_proj.weight module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.4.self_attention.linear_qkv.weight - module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight Params for bucket 15 (146800640 elements, 146800640 padded size): module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight Params for bucket 16 (103001600 elements, 103001600 padded size): - module.decoder.layers.3.self_attention.linear_qkv.weight - module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.linear_qkv.bias module.decoder.layers.3.self_attention.linear_proj.weight module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_qkv.weight Params for bucket 17 (146800640 elements, 146800640 padded size): module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight Params for bucket 18 (103001600 elements, 103001600 padded size): - module.decoder.layers.2.self_attention.linear_qkv.weight - module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.self_attention.linear_qkv.bias module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.bias module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.weight Params for bucket 19 (146800640 elements, 146800640 padded size): module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight Params for bucket 20 (103001600 elements, 103001600 padded size): - module.decoder.layers.1.self_attention.linear_qkv.weight - module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.1.mlp.router.weight - module.decoder.layers.1.self_attention.linear_qkv.bias module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.bias module.decoder.layers.1.self_attention.linear_proj.weight module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight Params for bucket 21 (146800640 elements, 146800640 padded size): module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight Params for bucket 22 (209890816 elements, 209890816 padded size): module.decoder.layers.0.mlp.router.weight - module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.0.self_attention.linear_qkv.weight module.decoder.layers.0.self_attention.linear_qkv.bias module.decoder.layers.0.pre_mlp_layernorm.weight - module.decoder.layers.0.self_attention.linear_proj.weight module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_proj.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 Params for bucket 1 (45875200 elements, 45875200 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 Params for bucket 2 (45875200 elements, 45875200 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight2 - module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight1 Params for bucket 3 (55050240 elements, 55050240 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight6 @@ -795,15 +795,15 @@ Params for bucket 4 (55050240 elements, 55050240 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight2 Params for bucket 5 (45875200 elements, 45875200 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight0 - module.decoder.layers.8.mlp.experts.linear_fc2.weight6 module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 module.decoder.layers.8.mlp.experts.linear_fc2.weight7 Params for bucket 6 (45875200 elements, 45875200 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight4 - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 module.decoder.layers.8.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 Params for bucket 7 (55050240 elements, 55050240 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight7 module.decoder.layers.8.mlp.experts.linear_fc1.weight5 @@ -817,11 +817,11 @@ Params for bucket 9 (45875200 elements, 45875200 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc1.weight0 Params for bucket 10 (45875200 elements, 45875200 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 module.decoder.layers.7.mlp.experts.linear_fc2.weight6 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 - module.decoder.layers.7.mlp.experts.linear_fc2.weight5 - module.decoder.layers.7.mlp.experts.linear_fc2.weight3 Params for bucket 11 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight1 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 @@ -836,16 +836,16 @@ Params for bucket 13 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight1 module.decoder.layers.7.mlp.experts.linear_fc1.weight0 Params for bucket 14 (45875200 elements, 45875200 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 - module.decoder.layers.6.mlp.experts.linear_fc2.weight3 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 Params for bucket 15 (45875200 elements, 45875200 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 module.decoder.layers.6.mlp.experts.linear_fc2.weight1 - module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 16 (55050240 elements, 55050240 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight6 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 @@ -856,15 +856,15 @@ Params for bucket 17 (55050240 elements, 55050240 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight1 Params for bucket 18 (45875200 elements, 45875200 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight0 - module.decoder.layers.5.mlp.experts.linear_fc2.weight6 module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 Params for bucket 19 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 module.decoder.layers.5.mlp.experts.linear_fc2.weight0 - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 Params for bucket 20 (55050240 elements, 55050240 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight7 module.decoder.layers.5.mlp.experts.linear_fc1.weight5 @@ -875,14 +875,14 @@ Params for bucket 21 (55050240 elements, 55050240 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight2 Params for bucket 22 (45875200 elements, 45875200 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight1 - module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 Params for bucket 23 (45875200 elements, 45875200 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 - module.decoder.layers.4.mlp.experts.linear_fc2.weight2 Params for bucket 24 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight1 module.decoder.layers.4.mlp.experts.linear_fc1.weight7 @@ -890,69 +890,69 @@ Params for bucket 24 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight6 Params for bucket 25 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight3 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 Params for bucket 26 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight2 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 Params for bucket 27 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 - module.decoder.layers.3.mlp.experts.linear_fc2.weight4 - module.decoder.layers.3.mlp.experts.linear_fc2.weight6 Params for bucket 28 (45875200 elements, 45875200 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight7 - module.decoder.layers.3.mlp.experts.linear_fc2.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 Params for bucket 29 (55050240 elements, 55050240 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight6 - module.decoder.layers.3.mlp.experts.linear_fc1.weight4 module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 Params for bucket 30 (55050240 elements, 55050240 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight3 - module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 Params for bucket 31 (45875200 elements, 45875200 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight0 - module.decoder.layers.2.mlp.experts.linear_fc2.weight7 module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 Params for bucket 32 (45875200 elements, 45875200 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight0 - module.decoder.layers.2.mlp.experts.linear_fc2.weight2 - module.decoder.layers.2.mlp.experts.linear_fc2.weight3 module.decoder.layers.2.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 Params for bucket 33 (55050240 elements, 55050240 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight7 - module.decoder.layers.2.mlp.experts.linear_fc1.weight6 module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 Params for bucket 34 (55050240 elements, 55050240 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight4 module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.2.mlp.experts.linear_fc1.weight3 Params for bucket 35 (45875200 elements, 45875200 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight1 - module.decoder.layers.2.mlp.experts.linear_fc1.weight0 module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 Params for bucket 36 (45875200 elements, 45875200 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight2 - module.decoder.layers.1.mlp.experts.linear_fc2.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight5 module.decoder.layers.1.mlp.experts.linear_fc2.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 Params for bucket 37 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight1 - module.decoder.layers.1.mlp.experts.linear_fc1.weight6 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 Params for bucket 38 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight5 - module.decoder.layers.1.mlp.experts.linear_fc1.weight3 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 Params for bucket 39 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight1 @@ -965,22 +965,22 @@ Params for bucket 40 (45875200 elements, 45875200 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight3 Params for bucket 41 (45875200 elements, 45875200 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight2 - module.decoder.layers.0.mlp.experts.linear_fc1.weight7 module.decoder.layers.0.mlp.experts.linear_fc2.weight1 module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 Params for bucket 42 (55050240 elements, 55050240 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight6 module.decoder.layers.0.mlp.experts.linear_fc1.weight5 module.decoder.layers.0.mlp.experts.linear_fc1.weight4 Params for bucket 43 (55050240 elements, 55050240 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight3 - module.decoder.layers.0.mlp.experts.linear_fc1.weight2 module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 Params for bucket 44 (18350080 elements, 18350080 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:51:10 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:03:17 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -988,7 +988,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1001,22 +1001,22 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42 INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 16:51:10 +[after dataloaders are built] datetime: 2026-08-26 18:03:17 done with setup ... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3520.33, 3527.19) - train/valid/test-data-iterators-setup ..........: (75.23, 90.08) + model-and-optimizer-setup ......................: (3457.01, 3462.94) + train/valid/test-data-iterators-setup ..........: (72.77, 88.58) training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-26 16:51:10 +[before the start of training step] datetime: 2026-08-26 18:03:17 /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. @@ -1059,10 +1059,6 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0... warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. - warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -1071,18 +1067,22 @@ Overwriting rerun_state_machine.current_iteration from -1 to 0... warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. - warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. @@ -1122,415 +1122,415 @@ WARNING:megatron.core.rerun_state_machine:Result validation enabled warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32512 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32949 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32949 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32512 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32951 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32485 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32485 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32997 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33009 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33009 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32951 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32997 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33009 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=1 calls=1 budget=40960 expected_recv=33009 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32724 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32949 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32724 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32958 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32958 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32949 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32590 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32590 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32727 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32727 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32994 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32868 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32994 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32868 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32501 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32999 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32999 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32501 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32758 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32999 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32758 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=2 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32565 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32590 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32427 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32565 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32590 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32427 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32512 expected_overflow=0 exact_recv_per_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32673 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32673 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=0 calls=1 budget=40960 expected_recv=32512 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32664 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32664 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33034 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33034 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32485 expected_overflow=0 exact_recv_per_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32402 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32402 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33034 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32958 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=1 calls=1 budget=40960 expected_recv=32485 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33034 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32958 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33023 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32447 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33023 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32447 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32469 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33023 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32533 expected_overflow=0 exact_recv_per_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32469 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32994 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=8 calls=1 budget=40960 expected_recv=33023 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=2 calls=1 budget=40960 expected_recv=32533 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32994 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33063 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32501 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=0 layer=9 calls=1 budget=40960 expected_recv=33063 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32501 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=3 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32565 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=4 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=5 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=7 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32565 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=6 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=8 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=0 layer=9 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=0 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=1 tokens_per_ep_rank=[32949, 32753, 32654, 32951, 32512, 32683, 32997, 32645] avg=32768.0000 max=32997 max_rank=6 min=32512 min_rank=4 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=2 tokens_per_ep_rank=[32762, 32841, 32861, 33009, 32485, 32746, 32665, 32775] avg=32768.0000 max=33009 max_rank=3 min=32485 min_rank=4 imbalance_ratio=1.007355 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[33098, 32596, 32727, 32724, 32531, 32963, 32718, 32787] avg=32768.0000 max=33098 max_rank=0 min=32531 min_rank=4 imbalance_ratio=1.010071 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=10 total_expected_recv=328141 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=3 tokens_per_ep_rank=[33096, 32596, 32727, 32724, 32533, 32963, 32718, 32787] avg=32768.0000 max=33096 max_rank=0 min=32533 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=2 max_load_reduction_pct=0.000060 moved_to_replicas=2 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33198, 32598, 32731, 32691, 32622] avg=32768.0000 max=33198 max_rank=3 min=32598 min_rank=4 imbalance_ratio=1.013123 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=4 tokens_per_ep_rank=[32788, 32807, 32709, 33096, 32700, 32731, 32691, 32622] avg=32768.0000 max=33096 max_rank=3 min=32622 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=102 max_load_reduction_pct=0.003072 moved_to_replicas=102 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=10 total_expected_recv=327931 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=10 total_expected_recv=326982 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77568 peak_demand=32868 required_capacity=1.003052 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=5 tokens_per_ep_rank=[32664, 32868, 32891, 32965, 32683, 32725, 32758, 32590] avg=32768.0000 max=32965 max_rank=3 min=32590 min_rank=7 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=10 total_expected_recv=328434 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33063 required_capacity=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=10 total_expected_recv=327040 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77440 peak_demand=32819 required_capacity=1.001556 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=6 tokens_per_ep_rank=[32958, 32765, 32806, 32999, 32681, 32822, 32673, 32440] avg=32768.0000 max=32999 max_rank=3 min=32440 min_rank=7 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=10 total_expected_recv=327943 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=32891 required_capacity=1.003754 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=7 tokens_per_ep_rank=[32994, 32777, 32775, 32427, 32642, 32908, 32812, 32809] avg=32768.0000 max=32994 max_rank=0 min=32427 min_rank=3 imbalance_ratio=1.006897 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32380, 33151, 32781, 33034, 32743] avg=32768.0000 max=33151 max_rank=4 min=32380 min_rank=3 imbalance_ratio=1.011688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=10 total_expected_recv=326929 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=0 records=10 total_expected_recv=327040 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77440 peak_demand=32819 required_capacity=1.001556 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=8 tokens_per_ep_rank=[32501, 32704, 32850, 32435, 33096, 32781, 33034, 32743] avg=32768.0000 max=33096 max_rank=4 min=32435 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=0 records=10 total_expected_recv=328040 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=32963 required_capacity=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=9 tokens_per_ep_rank=[32565, 32402, 32881, 32878, 32730, 32846, 33023, 32819] avg=32768.0000 max=33023 max_rank=6 min=32402 min_rank=1 imbalance_ratio=1.007782 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=0 records=10 total_expected_recv=327943 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=32891 required_capacity=1.003754 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=0 records=10 total_expected_recv=327931 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=0 records=10 total_expected_recv=326982 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77568 peak_demand=32868 required_capacity=1.003052 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=0 layer=10 tokens_per_ep_rank=[32864, 32469, 32789, 32447, 32867, 32835, 33063, 32810] avg=32768.0000 max=33063 max_rank=6 min=32447 min_rank=3 imbalance_ratio=1.009003 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=0 records=10 total_expected_recv=328434 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33063 required_capacity=1.009003 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=0 records=10 total_expected_recv=328141 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33096 required_capacity=1.010010 - [2026-08-26 16:51:35] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 25544.2 | throughput per GPU (TFLOP/s/GPU): 4.8 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.098005E+01 | loss scale: 1.0 | grad norm: 29.754 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33408 | min safe capacity: 1.0195 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=0 records=10 total_expected_recv=326929 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 18:03:43] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 25667.3 | throughput per GPU (TFLOP/s/GPU): 4.8 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.098005E+01 | loss scale: 1.0 | grad norm: 29.753 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33408 | min safe capacity: 1.0195 | Number of parameters in transformer block in billions: 20.11 Number of parameters in embedding layers in billions: 0.36 Total number of parameters in billions: 20.47 @@ -1553,4516 +1553,4516 @@ Theoretical memory footprints: weight and optimizer=146430.14 MB warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") /workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32873 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33011 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32617 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32819 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32873 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32873 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32738 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=0 calls=1 budget=40960 expected_recv=33011 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32617 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32819 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32873 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32738 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32903 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32903 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=0 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32970 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32970 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32903 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32903 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33005 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33005 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33005 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=2 calls=1 budget=40960 expected_recv=33005 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32692 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=1 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32692 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32635 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32942 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32635 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32755 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32577 expected_overflow=0 exact_recv_per_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32635 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33087 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32942 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32755 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33087 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=2 calls=1 budget=40960 expected_recv=32577 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33028 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33028 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=3 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33087 expected_overflow=0 exact_recv_per_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33028 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=3 calls=1 budget=40960 expected_recv=33087 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=4 calls=1 budget=40960 expected_recv=33028 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32942 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33017 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32942 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33017 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=4 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32389 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33017 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=5 calls=1 budget=40960 expected_recv=33017 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32389 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32958 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32958 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33024 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32702 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33024 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32071 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32702 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32466 expected_overflow=0 exact_recv_per_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33069 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32071 actual_valid_recv=32640 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=5 calls=1 budget=40960 expected_recv=32466 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33069 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32702 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32640 slack_slots=8320 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32702 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32958 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32389 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32958 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32389 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33077 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=6 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32990 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32464 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32990 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33077 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32464 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32502 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33024 expected_overflow=0 exact_recv_per_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33077 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32464 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32502 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=6 calls=1 budget=40960 expected_recv=33024 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32990 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32990 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32691 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32691 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=7 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32887 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32887 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32523 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32523 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32523 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=8 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32523 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32887 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32887 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=1 layer=9 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=1 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=1 tokens_per_ep_rank=[32779, 32819, 32738, 32617, 32873, 32739, 33011, 32568] avg=32768.0000 max=33011 max_rank=6 min=32568 min_rank=7 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=2 tokens_per_ep_rank=[32753, 32903, 32783, 32569, 32716, 32970, 32678, 32772] avg=32768.0000 max=32970 max_rank=5 min=32569 min_rank=3 imbalance_ratio=1.006165 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=3 tokens_per_ep_rank=[32577, 33005, 32812, 32747, 32773, 32838, 32700, 32692] avg=32768.0000 max=33005 max_rank=1 min=32577 min_rank=0 imbalance_ratio=1.007233 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=4 tokens_per_ep_rank=[33087, 32827, 32715, 32663, 32850, 32755, 32635, 32612] avg=32768.0000 max=33087 max_rank=0 min=32612 min_rank=7 imbalance_ratio=1.009735 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=5 tokens_per_ep_rank=[32739, 32790, 32942, 32691, 32709, 32649, 32596, 33028] avg=32768.0000 max=33028 max_rank=7 min=32596 min_rank=6 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=6 tokens_per_ep_rank=[32466, 33017, 32775, 32700, 32723, 32764, 32795, 32904] avg=32768.0000 max=33017 max_rank=1 min=32466 min_rank=0 imbalance_ratio=1.007599 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=7 tokens_per_ep_rank=[33077, 32389, 32854, 33024, 33069, 32702, 32958, 32071] avg=32768.0000 max=33077 max_rank=0 min=32071 min_rank=7 imbalance_ratio=1.009430 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=8 tokens_per_ep_rank=[32965, 32699, 32761, 32871, 32502, 32990, 32892, 32464] avg=32768.0000 max=32990 max_rank=5 min=32464 min_rank=7 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=9 tokens_per_ep_rank=[32594, 32706, 32809, 32911, 32679, 32826, 32928, 32691] avg=32768.0000 max=32928 max_rank=6 min=32594 min_rank=0 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=1 layer=10 tokens_per_ep_rank=[32867, 32618, 32887, 32815, 32841, 32523, 32797, 32796] avg=32768.0000 max=32887 max_rank=2 min=32523 min_rank=5 imbalance_ratio=1.003632 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=10 total_expected_recv=328076 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=32942 required_capacity=1.005310 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=10 total_expected_recv=326598 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77568 peak_demand=33028 required_capacity=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=10 total_expected_recv=327735 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=33069 required_capacity=1.009186 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=10 total_expected_recv=327773 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33017 required_capacity=1.007599 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=10 total_expected_recv=327756 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=32990 required_capacity=1.006775 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=1 records=10 total_expected_recv=327990 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=33011 required_capacity=1.007416 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=1 records=10 total_expected_recv=327608 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=33024 required_capacity=1.007812 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=1 records=10 total_expected_recv=327773 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33017 required_capacity=1.007599 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=1 records=10 total_expected_recv=326598 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77568 peak_demand=33028 required_capacity=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=1 records=10 total_expected_recv=327756 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=32990 required_capacity=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=1 records=10 total_expected_recv=328076 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=32942 required_capacity=1.005310 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=1 records=10 total_expected_recv=327735 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=33069 required_capacity=1.009186 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=1 records=10 total_expected_recv=327904 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76160 peak_demand=33087 required_capacity=1.009735 - [2026-08-26 16:51:36] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 322.9 | throughput per GPU (TFLOP/s/GPU): 382.4 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097681E+01 | loss scale: 1.0 | grad norm: 30.510 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33013 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] + [2026-08-26 18:03:43] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 323.6 | throughput per GPU (TFLOP/s/GPU): 381.7 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097681E+01 | loss scale: 1.0 | grad norm: 30.511 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32500 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32746 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33013 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33013 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32500 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32746 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33013 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=0 calls=1 budget=40960 expected_recv=32610 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32968 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32968 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32968 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32496 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32968 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32496 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=1 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32696 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32696 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33031 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32729 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32696 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33031 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32687 expected_overflow=0 exact_recv_per_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=2 calls=1 budget=40960 expected_recv=33031 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32729 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32696 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32687 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=2 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32587 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32982 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32587 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32982 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32587 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32707 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32587 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32982 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32909 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32909 expected_overflow=0 exact_recv_per_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32707 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32982 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32909 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=3 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32862 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32924 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32862 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32924 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32299 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32912 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32689 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32912 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32299 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32299 expected_overflow=0 exact_recv_per_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32912 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32689 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=4 calls=1 budget=40960 expected_recv=32299 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32923 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32923 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=5 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33036 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32553 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=6 calls=1 budget=40960 expected_recv=33036 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32553 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32553 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32567 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32567 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32553 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32567 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32567 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=6 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32748 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32883 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32748 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32883 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32934 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32491 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32805 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32934 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32883 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32934 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32491 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32805 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32883 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32934 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32878 expected_overflow=0 exact_recv_per_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=7 calls=1 budget=40960 expected_recv=32878 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32908 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32502 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32531 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32519 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32502 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32908 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32502 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32531 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32519 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32502 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32519 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32993 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32519 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32993 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32812 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32885 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32993 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=8 calls=1 budget=40960 expected_recv=32812 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32885 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32885 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32885 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=2 layer=9 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=2 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33121, 32475, 33013] avg=32768.0000 max=33121 max_rank=5 min=32475 min_rank=6 imbalance_ratio=1.010773 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=1 tokens_per_ep_rank=[32610, 32701, 32789, 32689, 32746, 33096, 32500, 33013] avg=32768.0000 max=33096 max_rank=5 min=32500 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=25 max_load_reduction_pct=0.000755 moved_to_replicas=25 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=2 tokens_per_ep_rank=[32826, 32642, 32496, 32979, 32620, 32968, 32803, 32810] avg=32768.0000 max=32979 max_rank=3 min=32496 min_rank=2 imbalance_ratio=1.006439 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=3 tokens_per_ep_rank=[32687, 32770, 33031, 32863, 32696, 32729, 32706, 32662] avg=32768.0000 max=33031 max_rank=2 min=32662 min_rank=7 imbalance_ratio=1.008026 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=4 tokens_per_ep_rank=[32909, 32776, 32707, 32644, 32768, 32982, 32771, 32587] avg=32768.0000 max=32982 max_rank=5 min=32587 min_rank=7 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=5 tokens_per_ep_rank=[32689, 32299, 32901, 32912, 32863, 32694, 32924, 32862] avg=32768.0000 max=32924 max_rank=6 min=32299 min_rank=1 imbalance_ratio=1.004761 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=6 tokens_per_ep_rank=[32923, 32612, 32782, 32709, 32739, 32705, 32775, 32899] avg=32768.0000 max=32923 max_rank=0 min=32612 min_rank=1 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=7 tokens_per_ep_rank=[32881, 32764, 32567, 32848, 32797, 32698, 32553, 33036] avg=32768.0000 max=33036 max_rank=7 min=32553 min_rank=6 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=8 tokens_per_ep_rank=[32699, 32934, 32491, 32805, 32878, 32883, 32706, 32748] avg=32768.0000 max=32934 max_rank=1 min=32491 min_rank=2 imbalance_ratio=1.005066 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[32457, 32519, 32502, 32908, 33170, 32812, 32797, 32979] avg=32768.0000 max=33170 max_rank=4 min=32457 min_rank=0 imbalance_ratio=1.012268 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=9 tokens_per_ep_rank=[32531, 32519, 32502, 32908, 33096, 32812, 32797, 32979] avg=32768.0000 max=33096 max_rank=4 min=32502 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=74 max_load_reduction_pct=0.002231 moved_to_replicas=74 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=10 total_expected_recv=328589 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33036 required_capacity=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=10 total_expected_recv=328020 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=10 total_expected_recv=328409 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=2 layer=10 tokens_per_ep_rank=[32615, 32608, 32627, 32757, 32817, 32842, 32885, 32993] avg=32768.0000 max=32993 max_rank=7 min=32608 min_rank=1 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=2 records=10 total_expected_recv=327420 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=32924 required_capacity=1.004761 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=10 total_expected_recv=328114 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75904 peak_demand=32979 required_capacity=1.006439 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=2 records=10 total_expected_recv=328589 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33036 required_capacity=1.008179 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=2 records=10 total_expected_recv=326893 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77696 peak_demand=33031 required_capacity=1.008026 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=10 total_expected_recv=326625 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78208 peak_demand=32934 required_capacity=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=2 records=10 total_expected_recv=328114 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75904 peak_demand=32979 required_capacity=1.006439 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=2 records=10 total_expected_recv=327370 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77312 peak_demand=32923 required_capacity=1.004730 - [2026-08-26 16:51:36] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 215.2 | throughput per GPU (TFLOP/s/GPU): 573.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.516744E+00 | loss scale: 1.0 | grad norm: 30.395 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=2 records=10 total_expected_recv=328409 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=2 records=10 total_expected_recv=326625 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78208 peak_demand=32934 required_capacity=1.005066 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=2 records=10 total_expected_recv=328020 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 18:03:43] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 214.4 | throughput per GPU (TFLOP/s/GPU): 575.9 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.516219E+00 | loss scale: 1.0 | grad norm: 30.358 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32555 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32980 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32555 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32980 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32643 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32555 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32538 expected_overflow=0 exact_recv_per_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32980 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32555 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=0 calls=1 budget=40960 expected_recv=32538 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32848 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32566 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32848 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32677 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32566 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32566 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32677 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32566 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=1 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32873 expected_overflow=0 exact_recv_per_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=2 calls=1 budget=40960 expected_recv=32873 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32501 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32838 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32982 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32501 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32501 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32838 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32982 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32982 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32501 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32982 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=3 calls=1 budget=40960 expected_recv=32900 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32546 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32499 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32546 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32499 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33006 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32546 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33006 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32499 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33006 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32930 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32499 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=4 calls=1 budget=40960 expected_recv=33006 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32930 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=4 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32715 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32825 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32715 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=5 calls=1 budget=40960 expected_recv=32825 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32681 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32681 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32794 expected_overflow=0 exact_recv_per_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32947 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=6 calls=1 budget=40960 expected_recv=32794 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32947 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32947 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32528 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32947 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32528 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33072 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33058 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33072 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32528 expected_overflow=0 exact_recv_per_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32592 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33058 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=7 calls=1 budget=40960 expected_recv=32528 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32592 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32592 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32497 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32592 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32497 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32532 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33072 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32532 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32650 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=8 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=8 calls=1 budget=40960 expected_recv=33072 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32829 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32650 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32829 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32532 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32532 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=3 layer=9 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=3 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33124, 32966, 32980, 32676, 32527] avg=32768.0000 max=33124 max_rank=3 min=32527 min_rank=7 imbalance_ratio=1.010864 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=1 tokens_per_ep_rank=[32538, 32690, 32643, 33096, 32966, 32980, 32676, 32555] avg=32768.0000 max=33096 max_rank=3 min=32538 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28 max_load_reduction_pct=0.000845 moved_to_replicas=28 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=2 tokens_per_ep_rank=[32895, 32566, 32965, 32685, 32684, 32824, 32848, 32677] avg=32768.0000 max=32965 max_rank=2 min=32566 min_rank=1 imbalance_ratio=1.006012 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=3 tokens_per_ep_rank=[32873, 32823, 32869, 32742, 32690, 32694, 32675, 32778] avg=32768.0000 max=32873 max_rank=0 min=32675 min_rank=6 imbalance_ratio=1.003204 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=4 tokens_per_ep_rank=[32900, 32797, 32501, 32476, 32982, 32838, 32778, 32872] avg=32768.0000 max=32982 max_rank=4 min=32476 min_rank=3 imbalance_ratio=1.006531 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=5 tokens_per_ep_rank=[32930, 32891, 32499, 32734, 33006, 32761, 32546, 32777] avg=32768.0000 max=33006 max_rank=4 min=32499 min_rank=2 imbalance_ratio=1.007263 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=6 tokens_per_ep_rank=[32825, 32715, 32628, 32826, 32753, 32780, 32920, 32697] avg=32768.0000 max=32920 max_rank=6 min=32628 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=7 tokens_per_ep_rank=[32794, 32681, 32796, 32652, 32623, 32925, 32840, 32833] avg=32768.0000 max=32925 max_rank=5 min=32623 min_rank=4 imbalance_ratio=1.004791 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=8 tokens_per_ep_rank=[32528, 32904, 32606, 32933, 32817, 32640, 32769, 32947] avg=32768.0000 max=32947 max_rank=7 min=32528 min_rank=0 imbalance_ratio=1.005463 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=9 tokens_per_ep_rank=[32558, 32800, 33072, 32851, 32497, 32592, 33058, 32716] avg=32768.0000 max=33072 max_rank=2 min=32497 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=3 layer=10 tokens_per_ep_rank=[32870, 32768, 32532, 32966, 32686, 32843, 32650, 32829] avg=32768.0000 max=32966 max_rank=3 min=32532 min_rank=2 imbalance_ratio=1.006042 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=10 total_expected_recv=327111 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77312 peak_demand=33072 required_capacity=1.009277 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=10 total_expected_recv=327760 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=33058 required_capacity=1.008850 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=10 total_expected_recv=327704 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=33006 required_capacity=1.007263 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=3 records=10 total_expected_recv=327877 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=32980 required_capacity=1.006470 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=3 records=10 total_expected_recv=327681 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32947 required_capacity=1.005463 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=10 total_expected_recv=327635 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=32904 required_capacity=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=3 records=10 total_expected_recv=327760 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=33058 required_capacity=1.008850 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=3 records=10 total_expected_recv=327961 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=3 records=10 total_expected_recv=327635 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=32904 required_capacity=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=3 records=10 total_expected_recv=327704 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=33006 required_capacity=1.007263 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=3 records=10 total_expected_recv=327111 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77312 peak_demand=33072 required_capacity=1.009277 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=3 records=10 total_expected_recv=327711 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=32930 required_capacity=1.004944 - [2026-08-26 16:51:36] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 213.6 | throughput per GPU (TFLOP/s/GPU): 578.0 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.118895E+01 | loss scale: 1.0 | grad norm: 107.913 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] + [2026-08-26 18:03:43] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 212.8 | throughput per GPU (TFLOP/s/GPU): 580.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.119224E+01 | loss scale: 1.0 | grad norm: 107.972 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32720 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32860 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32668 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32720 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32688 expected_overflow=0 exact_recv_per_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32860 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32668 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=0 calls=1 budget=40960 expected_recv=32688 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32997 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32997 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32972 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32997 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32670 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32972 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32972 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32670 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=1 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32986 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33060 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32765 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32986 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32545 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33060 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32765 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33060 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32545 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32986 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32545 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=2 calls=1 budget=40960 expected_recv=33060 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32986 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32545 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=2 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32508 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32509 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32849 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32508 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32509 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32849 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32560 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32560 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32849 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32964 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32849 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=3 calls=1 budget=40960 expected_recv=32964 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32932 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32932 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32502 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32502 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32859 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32502 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32859 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32502 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=4 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32883 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32883 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32542 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32542 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32883 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32518 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32883 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32542 expected_overflow=0 exact_recv_per_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32518 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33018 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=5 calls=1 budget=40960 expected_recv=32542 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32634 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33018 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32453 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32634 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32453 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32793 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33018 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32453 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32793 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=6 calls=1 budget=40960 expected_recv=33018 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32518 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32971 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32518 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32971 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32966 expected_overflow=0 exact_recv_per_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=6 calls=1 budget=40960 expected_recv=32966 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32666 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32666 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32918 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=7 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32918 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32956 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32956 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32918 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32956 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32918 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32797 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32797 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32853 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32465 expected_overflow=0 exact_recv_per_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32853 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32306 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32306 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32777 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=8 calls=1 budget=40960 expected_recv=32465 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32777 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32955 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32955 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32955 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32306 expected_overflow=0 exact_recv_per_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=4 layer=9 calls=1 budget=40960 expected_recv=32306 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=4 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=1 tokens_per_ep_rank=[32688, 32859, 32864, 32720, 32860, 32870, 32615, 32668] avg=32768.0000 max=32870 max_rank=5 min=32615 min_rank=6 imbalance_ratio=1.003113 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=2 tokens_per_ep_rank=[32717, 32796, 32569, 32997, 32679, 32670, 32744, 32972] avg=32768.0000 max=32997 max_rank=3 min=32569 min_rank=2 imbalance_ratio=1.006989 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=3 tokens_per_ep_rank=[32675, 33060, 32616, 32822, 32986, 32675, 32765, 32545] avg=32768.0000 max=33060 max_rank=1 min=32545 min_rank=7 imbalance_ratio=1.008911 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33156, 32509, 32745, 32560, 32448, 32849] avg=32768.0000 max=33156 max_rank=2 min=32448 min_rank=6 imbalance_ratio=1.011841 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=4 tokens_per_ep_rank=[32964, 32913, 33096, 32509, 32745, 32560, 32508, 32849] avg=32768.0000 max=33096 max_rank=2 min=32508 min_rank=6 imbalance_ratio=1.010010 max_load_reduction=60 max_load_reduction_pct=0.001810 moved_to_replicas=60 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=5 tokens_per_ep_rank=[32502, 32690, 32933, 32932, 32763, 32742, 32723, 32859] avg=32768.0000 max=32933 max_rank=2 min=32502 min_rank=0 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=6 tokens_per_ep_rank=[32782, 32883, 32542, 32679, 32920, 32782, 32797, 32759] avg=32768.0000 max=32920 max_rank=4 min=32542 min_rank=2 imbalance_ratio=1.004639 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=7 tokens_per_ep_rank=[32971, 33018, 32966, 32453, 32518, 32793, 32634, 32791] avg=32768.0000 max=33018 max_rank=1 min=32453 min_rank=3 imbalance_ratio=1.007629 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=8 tokens_per_ep_rank=[32954, 32697, 32700, 32666, 32757, 32911, 32753, 32706] avg=32768.0000 max=32954 max_rank=0 min=32666 min_rank=3 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=9 tokens_per_ep_rank=[32853, 32918, 32465, 32763, 32776, 32956, 32616, 32797] avg=32768.0000 max=32956 max_rank=5 min=32465 min_rank=2 imbalance_ratio=1.005737 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=10 total_expected_recv=327763 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=32986 required_capacity=1.006653 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=10 total_expected_recv=327782 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=32972 required_capacity=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=10 total_expected_recv=326955 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77824 peak_demand=32800 required_capacity=1.000977 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=10 total_expected_recv=327057 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77440 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=4 layer=10 tokens_per_ep_rank=[32955, 32898, 32306, 32813, 32759, 32777, 32800, 32836] avg=32768.0000 max=32955 max_rank=0 min=32306 min_rank=2 imbalance_ratio=1.005707 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=10 total_expected_recv=328732 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75264 peak_demand=33060 required_capacity=1.008911 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=4 records=10 total_expected_recv=327354 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=32997 required_capacity=1.006989 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=4 records=10 total_expected_recv=326955 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77824 peak_demand=32800 required_capacity=1.000977 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=4 records=10 total_expected_recv=327736 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=32956 required_capacity=1.005737 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=4 records=10 total_expected_recv=328732 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75264 peak_demand=33060 required_capacity=1.008911 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=4 records=10 total_expected_recv=327782 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=32972 required_capacity=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=4 records=10 total_expected_recv=327763 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=32986 required_capacity=1.006653 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=4 records=10 total_expected_recv=328061 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=32971 required_capacity=1.006195 - [2026-08-26 16:51:36] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 213.3 | throughput per GPU (TFLOP/s/GPU): 579.1 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.861985E+00 | loss scale: 1.0 | grad norm: 18.983 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=4 records=10 total_expected_recv=327057 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77440 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 18:03:44] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 212.6 | throughput per GPU (TFLOP/s/GPU): 580.7 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.859751E+00 | loss scale: 1.0 | grad norm: 18.912 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32706 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32716 expected_overflow=0 exact_recv_per_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32706 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=0 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=0 calls=1 budget=40960 expected_recv=32716 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32541 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32930 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32453 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33035 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32541 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32930 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32582 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33035 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32747 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32981 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32453 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33035 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32582 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32930 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32541 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=1 calls=1 budget=40960 expected_recv=33035 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32747 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32453 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32981 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32930 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32453 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=1 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32476 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32476 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=2 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32607 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32549 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32607 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32549 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32549 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33015 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33078 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32632 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32607 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33078 expected_overflow=0 exact_recv_per_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33015 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33078 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32632 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=3 calls=1 budget=40960 expected_recv=32607 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=3 calls=1 budget=40960 expected_recv=33078 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32946 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32547 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32946 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32547 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32946 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32946 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32547 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32669 expected_overflow=0 exact_recv_per_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32547 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=4 calls=1 budget=40960 expected_recv=32669 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33057 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33057 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32437 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32437 actual_valid_recv=32768 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32437 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32905 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32760 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33057 expected_overflow=0 exact_recv_per_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32437 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32905 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=5 calls=1 budget=40960 expected_recv=32760 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=5 calls=1 budget=40960 expected_recv=33057 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32892 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32580 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32767 expected_overflow=0 exact_recv_per_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32892 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32580 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=6 calls=1 budget=40960 expected_recv=32767 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=7 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32732 expected_overflow=0 exact_recv_per_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=7 calls=1 budget=40960 expected_recv=32732 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32800 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32537 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32537 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32800 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32890 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32647 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33033 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32890 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32890 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32537 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32647 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=8 calls=1 budget=40960 expected_recv=33033 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32890 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32537 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=8 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32856 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32856 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32875 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32875 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=5 layer=9 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=5 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32497, 33190, 32750, 32706, 32685, 32782] avg=32768.0000 max=33190 max_rank=3 min=32497 min_rank=2 imbalance_ratio=1.012878 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=1 tokens_per_ep_rank=[32716, 32818, 32591, 33096, 32750, 32706, 32685, 32782] avg=32768.0000 max=33096 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=94 max_load_reduction_pct=0.002832 moved_to_replicas=94 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=2 tokens_per_ep_rank=[32875, 32582, 32930, 32981, 32747, 33035, 32453, 32541] avg=32768.0000 max=33035 max_rank=5 min=32453 min_rank=6 imbalance_ratio=1.008148 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=3 tokens_per_ep_rank=[32881, 32743, 32678, 32963, 32723, 32817, 32863, 32476] avg=32768.0000 max=32963 max_rank=3 min=32476 min_rank=7 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=4 tokens_per_ep_rank=[33078, 32676, 32607, 33015, 32865, 32722, 32632, 32549] avg=32768.0000 max=33078 max_rank=0 min=32549 min_rank=7 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=5 tokens_per_ep_rank=[32741, 32770, 32669, 32886, 32946, 32763, 32547, 32822] avg=32768.0000 max=32946 max_rank=4 min=32547 min_rank=6 imbalance_ratio=1.005432 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=6 tokens_per_ep_rank=[32760, 32816, 33057, 32796, 32437, 32710, 32905, 32663] avg=32768.0000 max=33057 max_rank=2 min=32437 min_rank=4 imbalance_ratio=1.008820 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=7 tokens_per_ep_rank=[32678, 32911, 32767, 32895, 32892, 32745, 32580, 32676] avg=32768.0000 max=32911 max_rank=1 min=32580 min_rank=6 imbalance_ratio=1.004364 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33117, 32732, 32730, 32764, 32601, 32820, 32679] avg=32768.0000 max=33117 max_rank=1 min=32601 min_rank=5 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=8 tokens_per_ep_rank=[32701, 33096, 32732, 32730, 32764, 32622, 32820, 32679] avg=32768.0000 max=33096 max_rank=1 min=32622 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=9 tokens_per_ep_rank=[32807, 32890, 32824, 33033, 32537, 32606, 32647, 32800] avg=32768.0000 max=33033 max_rank=3 min=32537 min_rank=4 imbalance_ratio=1.008087 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=10 total_expected_recv=327673 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=33057 required_capacity=1.008820 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=5 layer=10 tokens_per_ep_rank=[32792, 32723, 32818, 32622, 32875, 32684, 32774, 32856] avg=32768.0000 max=32875 max_rank=4 min=32622 min_rank=3 imbalance_ratio=1.003265 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=5 records=10 total_expected_recv=326844 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77696 peak_demand=32856 required_capacity=1.002686 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=10 total_expected_recv=327536 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=32946 required_capacity=1.005432 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=5 records=10 total_expected_recv=327410 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77312 peak_demand=33035 required_capacity=1.008148 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=5 records=10 total_expected_recv=326906 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77824 peak_demand=32905 required_capacity=1.004181 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=10 total_expected_recv=329017 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75904 peak_demand=33096 required_capacity=1.010010 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=5 records=10 total_expected_recv=328025 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=33096 required_capacity=1.010010 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=5 records=10 total_expected_recv=328029 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=33078 required_capacity=1.009460 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=5 records=10 total_expected_recv=329017 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75904 peak_demand=33096 required_capacity=1.010010 - [2026-08-26 16:51:37] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 214.1 | throughput per GPU (TFLOP/s/GPU): 576.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.029423E+01 | loss scale: 1.0 | grad norm: 8.080 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=5 records=10 total_expected_recv=327536 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=32946 required_capacity=1.005432 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=5 records=10 total_expected_recv=327673 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=33057 required_capacity=1.008820 + [2026-08-26 18:03:44] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 212.2 | throughput per GPU (TFLOP/s/GPU): 581.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.028807E+01 | loss scale: 1.0 | grad norm: 8.100 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32622 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33010 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32622 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=0 calls=1 budget=40960 expected_recv=33010 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=0 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32555 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32440 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32935 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32977 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32555 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32881 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32440 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32718 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32977 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32935 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32977 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32881 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32718 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32977 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=1 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32584 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32584 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32584 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32573 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32923 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32584 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32923 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32573 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32923 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=2 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32935 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32935 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32935 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32338 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32935 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32811 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32957 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32338 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32873 expected_overflow=0 exact_recv_per_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32338 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32811 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32957 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32873 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32914 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=3 calls=1 budget=40960 expected_recv=32873 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32914 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32914 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32613 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32643 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32733 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32643 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32643 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=4 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33010 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32733 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33010 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=4 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=5 calls=1 budget=40960 expected_recv=33010 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32459 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32840 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32459 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32840 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32459 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32789 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32842 expected_overflow=0 exact_recv_per_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=5 calls=1 budget=40960 expected_recv=32842 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32789 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32761 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32907 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32538 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32761 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32907 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33094 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32907 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32630 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32538 expected_overflow=0 exact_recv_per_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=6 calls=1 budget=40960 expected_recv=33094 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32630 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=6 calls=1 budget=40960 expected_recv=32538 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32487 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32487 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32630 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32630 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33070 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32487 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=7 calls=1 budget=40960 expected_recv=33070 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32487 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32683 expected_overflow=0 exact_recv_per_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33001 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=7 calls=1 budget=40960 expected_recv=32683 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33001 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32829 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32717 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32829 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32717 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33001 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=8 calls=1 budget=40960 expected_recv=33001 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32658 expected_overflow=0 exact_recv_per_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32611 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=8 calls=1 budget=40960 expected_recv=32658 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32587 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32611 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32941 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32611 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32587 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32907 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32941 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32941 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32817 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32941 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32817 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32907 expected_overflow=0 exact_recv_per_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=6 layer=9 calls=1 budget=40960 expected_recv=32907 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=6 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=1 tokens_per_ep_rank=[32712, 32804, 33010, 32622, 32712, 32609, 32904, 32771] avg=32768.0000 max=33010 max_rank=2 min=32609 min_rank=5 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=2 tokens_per_ep_rank=[32698, 32940, 32881, 32935, 32440, 32718, 32977, 32555] avg=32768.0000 max=32977 max_rank=6 min=32440 min_rank=4 imbalance_ratio=1.006378 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=3 tokens_per_ep_rank=[32811, 32726, 32573, 32781, 32584, 32923, 32889, 32857] avg=32768.0000 max=32923 max_rank=5 min=32573 min_rank=2 imbalance_ratio=1.004730 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=4 tokens_per_ep_rank=[32873, 32338, 32811, 32719, 32935, 32957, 32749, 32762] avg=32768.0000 max=32957 max_rank=5 min=32338 min_rank=1 imbalance_ratio=1.005768 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33117, 32592, 32766, 32733, 32697, 32914] avg=32768.0000 max=33117 max_rank=2 min=32592 min_rank=3 imbalance_ratio=1.010651 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=5 tokens_per_ep_rank=[32682, 32643, 33096, 32613, 32766, 32733, 32697, 32914] avg=32768.0000 max=33096 max_rank=2 min=32613 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=21 max_load_reduction_pct=0.000634 moved_to_replicas=21 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=6 tokens_per_ep_rank=[32842, 32840, 32459, 32815, 32810, 32623, 32745, 33010] avg=32768.0000 max=33010 max_rank=7 min=32459 min_rank=2 imbalance_ratio=1.007385 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=7 tokens_per_ep_rank=[32538, 32761, 32717, 32662, 32676, 32907, 33094, 32789] avg=32768.0000 max=33094 max_rank=6 min=32538 min_rank=0 imbalance_ratio=1.009949 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=8 tokens_per_ep_rank=[32683, 32857, 32705, 32963, 32630, 33070, 32487, 32749] avg=32768.0000 max=33070 max_rank=5 min=32487 min_rank=6 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=9 tokens_per_ep_rank=[32658, 32790, 32829, 32753, 32717, 32790, 33001, 32606] avg=32768.0000 max=33001 max_rank=6 min=32606 min_rank=7 imbalance_ratio=1.007111 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=6 layer=10 tokens_per_ep_rank=[32907, 32587, 32611, 32780, 32778, 32941, 32817, 32723] avg=32768.0000 max=32941 max_rank=5 min=32587 min_rank=1 imbalance_ratio=1.005280 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=6 records=10 total_expected_recv=327736 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33010 required_capacity=1.007385 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=10 total_expected_recv=327048 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77440 peak_demand=32935 required_capacity=1.005096 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=10 total_expected_recv=327692 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=10 total_expected_recv=327286 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77696 peak_demand=32940 required_capacity=1.005249 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=10 total_expected_recv=328360 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=33094 required_capacity=1.009949 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=6 records=10 total_expected_recv=327643 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=32963 required_capacity=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=6 records=10 total_expected_recv=327286 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77696 peak_demand=32940 required_capacity=1.005249 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=6 records=10 total_expected_recv=328271 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33070 required_capacity=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=6 records=10 total_expected_recv=328360 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=33094 required_capacity=1.009949 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=6 records=10 total_expected_recv=327404 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=32907 required_capacity=1.004242 - [2026-08-26 16:51:37] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 212.4 | throughput per GPU (TFLOP/s/GPU): 581.4 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.212931E+00 | loss scale: 1.0 | grad norm: 4.980 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32551 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=6 records=10 total_expected_recv=327692 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=6 records=10 total_expected_recv=327048 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77440 peak_demand=32935 required_capacity=1.005096 + [2026-08-26 18:03:44] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 211.9 | throughput per GPU (TFLOP/s/GPU): 582.7 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.207916E+00 | loss scale: 1.0 | grad norm: 4.948 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32662 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32551 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32705 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32928 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32551 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32662 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32551 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32928 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32705 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=0 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32893 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32663 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32893 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32690 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32933 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32690 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32663 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32933 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=1 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32781 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32896 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32597 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32781 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32896 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32993 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32354 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32597 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32980 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32354 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32993 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32790 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32980 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32354 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32896 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32790 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32980 expected_overflow=0 exact_recv_per_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32896 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=2 calls=1 budget=40960 expected_recv=32980 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=3 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33792 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32603 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32603 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32679 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32743 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32827 expected_overflow=0 exact_recv_per_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32679 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32996 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32743 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=3 calls=1 budget=40960 expected_recv=32827 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32525 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32996 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32525 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32921 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32525 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32906 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32525 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32921 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32610 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32864 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32906 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32610 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32536 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32864 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32536 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32750 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32610 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=4 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32750 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32869 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32807 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32869 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32971 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32807 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32657 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32971 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32657 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32786 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32536 expected_overflow=0 exact_recv_per_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32501 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32768 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32786 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=5 calls=1 budget=40960 expected_recv=32536 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32501 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32917 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32917 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32625 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32917 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32625 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33036 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32482 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32625 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33042 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=6 calls=1 budget=40960 expected_recv=33036 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32482 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32665 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32979 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32665 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=6 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33042 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32979 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32721 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32684 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32721 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32684 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32501 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32815 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32984 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32501 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32815 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32984 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32482 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32596 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33042 expected_overflow=0 exact_recv_per_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32482 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=7 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32596 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=7 calls=1 budget=40960 expected_recv=33042 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32529 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32895 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32529 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32895 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32530 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33030 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32530 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33030 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32530 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=8 calls=1 budget=40960 expected_recv=33030 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32990 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32900 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32990 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32984 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32785 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32990 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32984 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32785 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=8 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32965 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32965 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32900 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32628 expected_overflow=0 exact_recv_per_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32900 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=7 layer=9 calls=1 budget=40960 expected_recv=32628 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=7 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=1 tokens_per_ep_rank=[32620, 32551, 32662, 32904, 32888, 32705, 32928, 32886] avg=32768.0000 max=32928 max_rank=6 min=32551 min_rank=1 imbalance_ratio=1.004883 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=2 tokens_per_ep_rank=[32721, 32663, 32893, 32675, 32891, 32678, 32933, 32690] avg=32768.0000 max=32933 max_rank=6 min=32663 min_rank=1 imbalance_ratio=1.005035 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=3 tokens_per_ep_rank=[32980, 32753, 32781, 32896, 32597, 32354, 32790, 32993] avg=32768.0000 max=32993 max_rank=7 min=32354 min_rank=5 imbalance_ratio=1.006866 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33188, 32743, 32603, 32985, 32587, 32591] avg=32768.0000 max=33188 max_rank=2 min=32587 min_rank=6 imbalance_ratio=1.012817 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=4 tokens_per_ep_rank=[32827, 32620, 33096, 32743, 32603, 32985, 32679, 32591] avg=32768.0000 max=33096 max_rank=2 min=32591 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=92 max_load_reduction_pct=0.002772 moved_to_replicas=92 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=5 tokens_per_ep_rank=[32610, 32906, 32996, 32596, 32525, 32726, 32864, 32921] avg=32768.0000 max=32996 max_rank=2 min=32525 min_rank=4 imbalance_ratio=1.006958 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=6 tokens_per_ep_rank=[32786, 32971, 32750, 32536, 32869, 32657, 32768, 32807] avg=32768.0000 max=32971 max_rank=1 min=32536 min_rank=3 imbalance_ratio=1.006195 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=7 tokens_per_ep_rank=[32665, 32625, 32501, 32804, 32917, 33036, 32826, 32770] avg=32768.0000 max=33036 max_rank=5 min=32501 min_rank=2 imbalance_ratio=1.008179 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=8 tokens_per_ep_rank=[32920, 32501, 32979, 33042, 32684, 32815, 32482, 32721] avg=32768.0000 max=33042 max_rank=3 min=32482 min_rank=6 imbalance_ratio=1.008362 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=9 tokens_per_ep_rank=[32785, 32530, 32596, 32795, 32895, 33030, 32984, 32529] avg=32768.0000 max=33030 max_rank=5 min=32529 min_rank=7 imbalance_ratio=1.007996 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=7 layer=10 tokens_per_ep_rank=[32623, 32965, 32990, 32628, 32572, 32642, 32900, 32824] avg=32768.0000 max=32990 max_rank=2 min=32572 min_rank=4 imbalance_ratio=1.006775 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=10 total_expected_recv=327619 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=33042 required_capacity=1.008362 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=10 total_expected_recv=328154 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=32984 required_capacity=1.006592 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=7 records=10 total_expected_recv=327732 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=32993 required_capacity=1.006866 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=7 records=10 total_expected_recv=328244 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33096 required_capacity=1.010010 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=7 records=10 total_expected_recv=327441 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=32917 required_capacity=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=10 total_expected_recv=327537 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=32980 required_capacity=1.006470 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=7 records=10 total_expected_recv=327085 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77312 peak_demand=32971 required_capacity=1.006195 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=7 records=10 total_expected_recv=327628 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=33036 required_capacity=1.008179 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=7 records=10 total_expected_recv=328154 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=32984 required_capacity=1.006592 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=7 records=10 total_expected_recv=327619 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=33042 required_capacity=1.008362 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=7 records=10 total_expected_recv=327537 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=32980 required_capacity=1.006470 - [2026-08-26 16:51:37] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 212.4 | throughput per GPU (TFLOP/s/GPU): 581.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.779124E+00 | loss scale: 1.0 | grad norm: 5.067 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32573 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] + [2026-08-26 18:03:44] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 211.7 | throughput per GPU (TFLOP/s/GPU): 583.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.788445E+00 | loss scale: 1.0 | grad norm: 5.800 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32835 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32861 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32922 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32913 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33078 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32226 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32573 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33078 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32573 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32835 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32861 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32922 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32913 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32226 actual_valid_recv=32512 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=0 calls=1 budget=40960 expected_recv=33078 actual_valid_recv=33792 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32226 actual_valid_recv=32512 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32573 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33792 slack_slots=7168 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=0 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=32512 slack_slots=8448 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=1 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32619 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32619 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32562 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32701 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=1 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32562 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32701 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32883 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32983 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32615 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32983 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32883 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32615 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32983 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32562 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32685 expected_overflow=0 exact_recv_per_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32562 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32839 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=2 calls=1 budget=40960 expected_recv=32685 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32839 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32535 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32985 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32535 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32985 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=3 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32697 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32649 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32886 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32697 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32742 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32649 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32574 expected_overflow=0 exact_recv_per_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32886 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32574 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33045 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32742 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=4 calls=1 budget=40960 expected_recv=32574 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33045 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32783 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32648 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32644 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33045 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32783 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32648 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32644 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33045 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33034 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33034 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32569 expected_overflow=0 exact_recv_per_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32676 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=5 calls=1 budget=40960 expected_recv=33034 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=5 calls=1 budget=40960 expected_recv=32569 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32902 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32676 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32902 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32988 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32902 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32988 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32788 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32988 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32902 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32788 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33028 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33028 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=6 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=7 calls=1 budget=40960 expected_recv=33028 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32916 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32916 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32651 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32651 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32431 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32804 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32431 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32898 expected_overflow=0 exact_recv_per_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32804 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=7 calls=1 budget=40960 expected_recv=32898 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32612 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32504 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32612 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32504 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33067 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32871 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33067 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32537 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32871 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32537 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=8 calls=1 budget=40960 expected_recv=33067 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32745 expected_overflow=0 exact_recv_per_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32537 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=8 calls=1 budget=40960 expected_recv=32745 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32562 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32562 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33051 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32723 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=8 layer=9 calls=1 budget=40960 expected_recv=33051 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32723 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=8 layer=9 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=8 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=1 tokens_per_ep_rank=[32736, 33078, 32922, 32913, 32573, 32226, 32861, 32835] avg=32768.0000 max=33078 max_rank=1 min=32226 min_rank=5 imbalance_ratio=1.009460 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32561, 32608, 32619, 32778, 33175] avg=32768.0000 max=33175 max_rank=7 min=32561 min_rank=3 imbalance_ratio=1.012421 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=2 tokens_per_ep_rank=[32845, 32725, 32833, 32640, 32608, 32619, 32778, 33096] avg=32768.0000 max=33096 max_rank=7 min=32608 min_rank=4 imbalance_ratio=1.010010 max_load_reduction=79 max_load_reduction_pct=0.002381 moved_to_replicas=79 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=3 tokens_per_ep_rank=[32685, 32883, 32615, 32752, 32562, 32983, 32963, 32701] avg=32768.0000 max=32983 max_rank=5 min=32562 min_rank=4 imbalance_ratio=1.006561 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=4 tokens_per_ep_rank=[32704, 32535, 32850, 32985, 32655, 32710, 32866, 32839] avg=32768.0000 max=32985 max_rank=3 min=32535 min_rank=1 imbalance_ratio=1.006622 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=5 tokens_per_ep_rank=[32574, 32649, 32886, 32841, 32742, 32963, 32697, 32792] avg=32768.0000 max=32963 max_rank=5 min=32574 min_rank=0 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=6 tokens_per_ep_rank=[33034, 32644, 33045, 32642, 32569, 32648, 32783, 32779] avg=32768.0000 max=33045 max_rank=2 min=32569 min_rank=4 imbalance_ratio=1.008453 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=7 tokens_per_ep_rank=[32659, 32988, 32788, 32902, 32609, 32709, 32813, 32676] avg=32768.0000 max=32988 max_rank=1 min=32609 min_rank=4 imbalance_ratio=1.006714 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=8 tokens_per_ep_rank=[32431, 32772, 32726, 32651, 32898, 32722, 32916, 33028] avg=32768.0000 max=33028 max_rank=7 min=32431 min_rank=0 imbalance_ratio=1.007935 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=9 tokens_per_ep_rank=[33067, 32796, 32871, 32745, 32745, 32504, 32612, 32804] avg=32768.0000 max=33067 max_rank=0 min=32504 min_rank=5 imbalance_ratio=1.009125 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33173, 32460] avg=32768.0000 max=33173 max_rank=6 min=32460 min_rank=7 imbalance_ratio=1.012360 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=8 layer=10 tokens_per_ep_rank=[33051, 32723, 32667, 32642, 32866, 32562, 33096, 32537] avg=32768.0000 max=33096 max_rank=6 min=32537 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=77 max_load_reduction_pct=0.002321 moved_to_replicas=77 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=8 records=10 total_expected_recv=328087 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=10 total_expected_recv=328203 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=33045 required_capacity=1.008453 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=10 total_expected_recv=326827 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78464 peak_demand=32898 required_capacity=1.003967 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=10 total_expected_recv=326646 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78592 peak_demand=32983 required_capacity=1.006561 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=10 total_expected_recv=327713 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32985 required_capacity=1.006622 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=8 records=10 total_expected_recv=328385 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=8 records=10 total_expected_recv=326646 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78592 peak_demand=32983 required_capacity=1.006561 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=8 records=10 total_expected_recv=327793 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75904 peak_demand=33078 required_capacity=1.009460 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=8 records=10 total_expected_recv=327786 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=33067 required_capacity=1.009125 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=8 records=10 total_expected_recv=327713 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32985 required_capacity=1.006622 - [2026-08-26 16:51:37] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 212.3 | throughput per GPU (TFLOP/s/GPU): 581.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.615476E+00 | loss scale: 1.0 | grad norm: 3.931 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=8 records=10 total_expected_recv=328203 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=33045 required_capacity=1.008453 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=8 records=10 total_expected_recv=326827 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78464 peak_demand=32898 required_capacity=1.003967 + [2026-08-26 18:03:44] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 212.3 | throughput per GPU (TFLOP/s/GPU): 581.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.620694E+00 | loss scale: 1.0 | grad norm: 3.976 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32680 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32821 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32876 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32999 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32680 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32821 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32876 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32432 expected_overflow=0 exact_recv_per_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32999 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=0 calls=1 budget=40960 expected_recv=32432 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32879 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32771 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32879 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32579 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32579 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32771 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32579 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32693 expected_overflow=0 exact_recv_per_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=1 calls=1 budget=40960 expected_recv=32693 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33022 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33022 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=2 calls=1 budget=40960 expected_recv=33022 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32872 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32872 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32603 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32851 expected_overflow=0 exact_recv_per_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32603 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=2 calls=1 budget=40960 expected_recv=32851 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32505 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32854 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32505 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32854 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32704 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32700 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32809 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32943 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32784 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32704 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32700 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32809 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32943 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32784 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=3 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32846 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32846 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32770 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32620 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32710 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32770 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32620 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32888 expected_overflow=0 exact_recv_per_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32710 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32888 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=4 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33041 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33041 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32787 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32290 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32787 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32822 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32631 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32290 actual_valid_recv=32768 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32843 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32631 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32843 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33070 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32822 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32290 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32631 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33070 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33041 expected_overflow=0 exact_recv_per_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=5 calls=1 budget=40960 expected_recv=32290 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=5 calls=1 budget=40960 expected_recv=33041 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32725 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32725 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33070 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32588 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32889 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=6 calls=1 budget=40960 expected_recv=33070 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32656 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32588 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32726 expected_overflow=0 exact_recv_per_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32889 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32726 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32885 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32885 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=6 calls=1 budget=40960 expected_recv=32656 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32773 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32773 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32751 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32606 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32836 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32820 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32885 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32751 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32606 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32836 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32820 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32885 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32682 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32769 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=7 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32769 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32682 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32939 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32568 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32939 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32627 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32568 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32538 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32627 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32803 expected_overflow=0 exact_recv_per_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32774 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32538 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=8 calls=1 budget=40960 expected_recv=32803 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32774 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32782 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32782 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32824 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32538 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32728 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32824 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32753 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32538 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32728 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32806 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32753 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32939 expected_overflow=0 exact_recv_per_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32806 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=9 layer=9 calls=1 budget=40960 expected_recv=32939 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=9 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=1 tokens_per_ep_rank=[32730, 32680, 32954, 32876, 32652, 32432, 32821, 32999] avg=32768.0000 max=32999 max_rank=7 min=32432 min_rank=5 imbalance_ratio=1.007050 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=2 tokens_per_ep_rank=[32579, 32879, 32866, 32870, 32722, 32693, 32764, 32771] avg=32768.0000 max=32879 max_rank=1 min=32579 min_rank=0 imbalance_ratio=1.003387 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=3 tokens_per_ep_rank=[32603, 33022, 32872, 32770, 32851, 32710, 32678, 32638] avg=32768.0000 max=33022 max_rank=1 min=32603 min_rank=0 imbalance_ratio=1.007751 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=4 tokens_per_ep_rank=[32943, 32854, 32704, 32700, 32784, 32845, 32505, 32809] avg=32768.0000 max=32943 max_rank=0 min=32505 min_rank=6 imbalance_ratio=1.005341 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=5 tokens_per_ep_rank=[32710, 32846, 32787, 32620, 32929, 32888, 32594, 32770] avg=32768.0000 max=32929 max_rank=4 min=32594 min_rank=6 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=6 tokens_per_ep_rank=[32631, 32920, 32810, 32290, 33041, 32843, 32787, 32822] avg=32768.0000 max=33041 max_rank=4 min=32290 min_rank=3 imbalance_ratio=1.008331 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=7 tokens_per_ep_rank=[32889, 32708, 33070, 32588, 32656, 32726, 32782, 32725] avg=32768.0000 max=33070 max_rank=2 min=32588 min_rank=3 imbalance_ratio=1.009216 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=8 tokens_per_ep_rank=[32606, 32773, 32885, 32820, 32616, 32836, 32857, 32751] avg=32768.0000 max=32885 max_rank=2 min=32606 min_rank=0 imbalance_ratio=1.003571 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32540, 33183, 32803, 32808, 32769, 32568] avg=32768.0000 max=33183 max_rank=3 min=32540 min_rank=2 imbalance_ratio=1.012665 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=9 tokens_per_ep_rank=[32791, 32682, 32627, 33096, 32803, 32808, 32769, 32568] avg=32768.0000 max=33096 max_rank=3 min=32568 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=87 max_load_reduction_pct=0.002622 moved_to_replicas=87 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=10 total_expected_recv=327993 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33041 required_capacity=1.008331 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=10 total_expected_recv=328381 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75776 peak_demand=33070 required_capacity=1.009216 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=10 total_expected_recv=327391 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32999 required_capacity=1.007050 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=10 total_expected_recv=328146 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=33022 required_capacity=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=9 layer=10 tokens_per_ep_rank=[32753, 32782, 32806, 32728, 32939, 32824, 32774, 32538] avg=32768.0000 max=32939 max_rank=4 min=32538 min_rank=7 imbalance_ratio=1.005219 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=9 records=10 total_expected_recv=327331 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77184 peak_demand=32857 required_capacity=1.002716 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=9 records=10 total_expected_recv=328146 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76288 peak_demand=33022 required_capacity=1.007751 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=10 total_expected_recv=327605 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32888 required_capacity=1.003662 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=9 records=10 total_expected_recv=327358 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33096 required_capacity=1.010010 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=9 records=10 total_expected_recv=327391 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32999 required_capacity=1.007050 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=9 records=10 total_expected_recv=327235 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77312 peak_demand=32943 required_capacity=1.005341 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=9 records=10 total_expected_recv=327605 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32888 required_capacity=1.003662 - [2026-08-26 16:51:37] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 212.0 | throughput per GPU (TFLOP/s/GPU): 582.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.545881E+00 | loss scale: 1.0 | grad norm: 3.683 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=9 records=10 total_expected_recv=328381 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=75776 peak_demand=33070 required_capacity=1.009216 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=9 records=10 total_expected_recv=327993 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33041 required_capacity=1.008331 + [2026-08-26 18:03:45] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 212.2 | throughput per GPU (TFLOP/s/GPU): 582.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.552052E+00 | loss scale: 1.0 | grad norm: 3.717 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32808 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32659 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32823 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32801 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32630 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32659 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32863 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32826 expected_overflow=0 exact_recv_per_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32808 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32823 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33664 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32801 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32630 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32863 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=0 calls=1 budget=40960 expected_recv=32826 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32882 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32845 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32882 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32882 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32616 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32575 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32845 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32882 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32616 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32917 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32575 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32917 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32667 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32917 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32575 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32667 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32867 expected_overflow=0 exact_recv_per_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32575 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32883 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=1 calls=1 budget=40960 expected_recv=32867 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32954 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32894 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32540 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32883 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32954 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32894 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32392 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32540 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32392 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32752 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32540 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32818 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32540 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32752 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32818 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32392 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32911 expected_overflow=0 exact_recv_per_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32392 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=2 calls=1 budget=40960 expected_recv=32911 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32597 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32597 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32816 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32735 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32847 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33012 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32816 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32735 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32847 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=3 calls=1 budget=40960 expected_recv=33012 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32847 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32652 expected_overflow=0 exact_recv_per_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32847 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32903 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=3 calls=1 budget=40960 expected_recv=32652 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32903 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32929 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32929 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32698 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32698 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32558 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32924 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32766 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32558 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32924 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32766 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32924 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32866 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32862 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32757 expected_overflow=0 exact_recv_per_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32654 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32862 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32866 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=4 calls=1 budget=40960 expected_recv=32757 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32796 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32654 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32796 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32591 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32763 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32591 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32901 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32763 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32711 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32901 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32922 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32711 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32967 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32862 expected_overflow=0 exact_recv_per_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32922 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32607 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32967 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=5 calls=1 budget=40960 expected_recv=32862 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32607 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32594 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32699 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32594 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32903 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32699 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32813 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32509 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32903 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32813 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32903 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32509 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32762 expected_overflow=0 exact_recv_per_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32509 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32638 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32509 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=6 calls=1 budget=40960 expected_recv=32762 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32578 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32578 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32638 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33053 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32828 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=7 calls=1 budget=40960 expected_recv=33053 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32736 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32828 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32578 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32736 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32734 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32578 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32425 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32734 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32425 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32899 expected_overflow=0 exact_recv_per_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32425 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32870 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=7 calls=1 budget=40960 expected_recv=32899 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32870 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32764 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32764 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32830 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32830 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32552 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32552 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32925 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32841 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32494 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32552 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32963 expected_overflow=0 exact_recv_per_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32925 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32841 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32494 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33011 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=8 calls=1 budget=40960 expected_recv=32963 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32562 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33010 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33011 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33010 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32562 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33010 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32494 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32562 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32494 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32562 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32749 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=10 layer=9 calls=1 budget=40960 expected_recv=32749 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33011 expected_overflow=0 exact_recv_per_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=10 layer=9 calls=1 budget=40960 expected_recv=33011 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=10 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=1 tokens_per_ep_rank=[32863, 32801, 32734, 32630, 32826, 32823, 32659, 32808] avg=32768.0000 max=32863 max_rank=0 min=32630 min_rank=3 imbalance_ratio=1.002899 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=2 tokens_per_ep_rank=[32575, 32917, 32775, 32667, 32867, 32616, 32845, 32882] avg=32768.0000 max=32917 max_rank=1 min=32575 min_rank=0 imbalance_ratio=1.004547 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=3 tokens_per_ep_rank=[32392, 32752, 32540, 32818, 32911, 32894, 32883, 32954] avg=32768.0000 max=32954 max_rank=7 min=32392 min_rank=0 imbalance_ratio=1.005676 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=4 tokens_per_ep_rank=[32847, 32776, 33012, 32709, 32652, 32735, 32816, 32597] avg=32768.0000 max=33012 max_rank=2 min=32597 min_rank=7 imbalance_ratio=1.007446 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=5 tokens_per_ep_rank=[32609, 32766, 32558, 32924, 32757, 32698, 32929, 32903] avg=32768.0000 max=32929 max_rank=6 min=32558 min_rank=2 imbalance_ratio=1.004913 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=6 tokens_per_ep_rank=[32711, 32763, 32591, 32901, 32862, 32796, 32654, 32866] avg=32768.0000 max=32901 max_rank=3 min=32591 min_rank=2 imbalance_ratio=1.004059 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=7 tokens_per_ep_rank=[32813, 32699, 32594, 32780, 32762, 32607, 32967, 32922] avg=32768.0000 max=32967 max_rank=6 min=32594 min_rank=2 imbalance_ratio=1.006073 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=8 tokens_per_ep_rank=[33053, 32578, 32736, 32828, 32899, 32638, 32509, 32903] avg=32768.0000 max=33053 max_rank=0 min=32509 min_rank=6 imbalance_ratio=1.008698 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=10 total_expected_recv=328248 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32963 required_capacity=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=9 tokens_per_ep_rank=[32678, 32880, 32830, 32764, 32963, 32870, 32425, 32734] avg=32768.0000 max=32963 max_rank=4 min=32425 min_rank=6 imbalance_ratio=1.005951 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=10 layer=10 tokens_per_ep_rank=[32562, 33011, 32494, 33010, 32749, 32841, 32925, 32552] avg=32768.0000 max=33011 max_rank=1 min=32494 min_rank=2 imbalance_ratio=1.007416 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=10 records=10 total_expected_recv=328121 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32954 required_capacity=1.005676 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=10 total_expected_recv=326864 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78208 peak_demand=33012 required_capacity=1.007446 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=10 total_expected_recv=327518 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=32894 required_capacity=1.003845 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=10 records=10 total_expected_recv=327612 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=32967 required_capacity=1.006073 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=10 total_expected_recv=327943 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33011 required_capacity=1.007416 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=10 records=10 total_expected_recv=327518 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=32894 required_capacity=1.003845 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=10 records=10 total_expected_recv=328031 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76032 peak_demand=33010 required_capacity=1.007385 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=10 records=10 total_expected_recv=327103 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77056 peak_demand=33053 required_capacity=1.008698 - [2026-08-26 16:51:38] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 621.9 | throughput per GPU (TFLOP/s/GPU): 198.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.506625E+00 | loss scale: 1.0 | grad norm: 3.481 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=10 records=10 total_expected_recv=326864 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=78208 peak_demand=33012 required_capacity=1.007446 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=10 records=10 total_expected_recv=328248 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=32963 required_capacity=1.005951 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=10 records=10 total_expected_recv=327943 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33011 required_capacity=1.007416 + [2026-08-26 18:03:45] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 637.6 | throughput per GPU (TFLOP/s/GPU): 193.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.513948E+00 | loss scale: 1.0 | grad norm: 3.513 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32640 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32621 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32613 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32940 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32759 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32972 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32640 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32719 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32621 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32940 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32613 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32880 expected_overflow=0 exact_recv_per_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32759 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32972 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32719 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32613 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=0 calls=1 budget=40960 expected_recv=32880 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=0 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32435 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32722 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32891 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32435 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32779 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32891 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32722 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32776 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32778 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32779 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32953 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32810 expected_overflow=0 exact_recv_per_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32778 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32776 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32953 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=1 calls=1 budget=40960 expected_recv=32810 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32686 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=1 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32686 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33056 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32791 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=2 calls=1 budget=40960 expected_recv=33056 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32791 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32694 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32739 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32642 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32795 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32694 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32739 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32741 expected_overflow=0 exact_recv_per_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32642 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32560 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32795 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33058 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=2 calls=1 budget=40960 expected_recv=32741 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32560 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33058 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33015 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=2 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33058 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33015 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32565 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33015 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32904 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32565 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=3 calls=1 budget=40960 expected_recv=33015 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32560 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32934 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32904 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32463 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32934 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32463 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32934 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32560 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32645 expected_overflow=0 exact_recv_per_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32934 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=3 calls=1 budget=40960 expected_recv=32645 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32857 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32572 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=3 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32572 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32865 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32857 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32780 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32865 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32737 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32844 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32737 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32968 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32780 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32844 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32968 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32709 expected_overflow=0 exact_recv_per_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32564 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=4 calls=1 budget=40960 expected_recv=32709 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32564 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=4 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32472 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32550 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32775 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32472 actual_valid_recv=32896 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32550 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32968 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32775 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32623 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32968 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32731 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=5 calls=1 budget=40960 expected_recv=32623 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=5 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32731 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=5 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=6 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32829 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32635 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33065 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32829 actual_valid_recv=33280 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32730 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32655 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32635 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33065 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32618 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32730 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32655 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32712 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32850 expected_overflow=0 exact_recv_per_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32618 actual_valid_recv=33024 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32712 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33068 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=6 calls=1 budget=40960 expected_recv=32850 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32744 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33068 actual_valid_recv=33536 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=6 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32744 actual_valid_recv=33152 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32516 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32708 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32890 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32909 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32516 actual_valid_recv=32896 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32708 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32890 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32909 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32441 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33065 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32890 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32608 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32441 actual_valid_recv=32768 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33065 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=7 calls=1 budget=40960 expected_recv=32890 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33068 expected_overflow=0 exact_recv_per_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32608 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32456 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=32768 slack_slots=8192 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=7 calls=1 budget=40960 expected_recv=33068 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32909 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32456 actual_valid_recv=33024 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33280 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32696 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=7 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32868 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32909 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32696 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32833 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32868 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32678 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32609 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=8 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33536 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32833 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32868 actual_valid_recv=33152 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33536 slack_slots=7424 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32678 actual_valid_recv=33152 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32456 expected_overflow=0 exact_recv_per_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32609 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33152 slack_slots=7808 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=7 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=8 calls=1 budget=40960 expected_recv=32456 actual_valid_recv=33024 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32675 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32211 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33069 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33096 actual_valid_recv=33664 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=8 calls=1 received_num_tokens=40960 valid_num_tokens=33024 slack_slots=7936 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32675 actual_valid_recv=33280 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32211 actual_valid_recv=32896 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=6 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33664 slack_slots=7296 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=3 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33280 slack_slots=7680 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33069 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32792 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32920 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 iteration=11 layer=9 calls=1 budget=40960 expected_recv=33069 actual_valid_recv=33408 observed_loss=0 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32792 actual_valid_recv=33408 observed_loss=0 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32920 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=5 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=32896 slack_slots=8064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=0 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32772 expected_overflow=0 exact_recv_per_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=4 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=1 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 iteration=11 layer=9 calls=1 budget=40960 expected_recv=32772 actual_valid_recv=33408 observed_loss=0 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][experts_pre] global_rank=2 iteration=11 layer=9 calls=1 received_num_tokens=40960 valid_num_tokens=33408 slack_slots=7552 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=1 tokens_per_ep_rank=[32880, 32719, 32613, 32940, 32759, 32972, 32640, 32621] avg=32768.0000 max=32972 max_rank=5 min=32613 min_rank=2 imbalance_ratio=1.006226 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=2 tokens_per_ep_rank=[32953, 32778, 32810, 32891, 32779, 32776, 32435, 32722] avg=32768.0000 max=32953 max_rank=0 min=32435 min_rank=6 imbalance_ratio=1.005646 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=3 tokens_per_ep_rank=[32795, 32739, 32741, 33056, 32694, 32642, 32686, 32791] avg=32768.0000 max=33056 max_rank=3 min=32642 min_rank=5 imbalance_ratio=1.008789 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=4 tokens_per_ep_rank=[32934, 32560, 32645, 32565, 32904, 32463, 33058, 33015] avg=32768.0000 max=33058 max_rank=6 min=32463 min_rank=5 imbalance_ratio=1.008850 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=5 tokens_per_ep_rank=[32844, 32780, 32709, 32780, 32865, 32737, 32572, 32857] avg=32768.0000 max=32865 max_rank=4 min=32572 min_rank=6 imbalance_ratio=1.002960 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[32432, 32968, 33118, 32450, 32775, 32550, 33287, 32564] avg=32768.0000 max=33287 max_rank=6 min=32432 min_rank=0 imbalance_ratio=1.015839 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=6 tokens_per_ep_rank=[32623, 32968, 33096, 32472, 32775, 32550, 33096, 32564] avg=32768.0000 max=33096 max_rank=2 min=32472 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=191 max_load_reduction_pct=0.005738 moved_to_replicas=213 active_clone_links=2 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32580, 33151, 32731] avg=32768.0000 max=33151 max_rank=6 min=32580 min_rank=5 imbalance_ratio=1.011688 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=7 tokens_per_ep_rank=[32618, 32655, 32850, 32829, 32730, 32635, 33096, 32731] avg=32768.0000 max=33096 max_rank=6 min=32618 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=55 max_load_reduction_pct=0.001659 moved_to_replicas=55 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=8 tokens_per_ep_rank=[32890, 33065, 33068, 32516, 32441, 32708, 32744, 32712] avg=32768.0000 max=33068 max_rank=2 min=32441 min_rank=4 imbalance_ratio=1.009155 max_load_reduction=0 max_load_reduction_pct=0.000000 moved_to_replicas=0 active_clone_links=0 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=7 ep_rank=7 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32407, 32696, 32833, 33145, 32909, 32608] avg=32768.0000 max=33145 max_rank=5 min=32407 min_rank=2 imbalance_ratio=1.011505 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=6 ep_rank=6 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 -WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=5 ep_rank=5 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=9 tokens_per_ep_rank=[32868, 32678, 32456, 32696, 32833, 33096, 32909, 32608] avg=32768.0000 max=33096 max_rank=5 min=32456 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=49 max_load_reduction_pct=0.001478 moved_to_replicas=49 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=3 ep_rank=3 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=0 ep_rank=0 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][pre_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32199, 33108, 32609] avg=32768.0000 max=33108 max_rank=6 min=32199 min_rank=5 imbalance_ratio=1.010376 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=1 ep_rank=1 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=4 ep_rank=4 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 +WARNING:megatron.training.drop_debug_utils:[MOE_ULTRAEP_DEBUG][post_balance] global_rank=2 ep_rank=2 iteration=11 layer=10 tokens_per_ep_rank=[33069, 32920, 32772, 32675, 32792, 32211, 33096, 32609] avg=32768.0000 max=33096 max_rank=6 min=32211 min_rank=5 imbalance_ratio=1.010010 max_load_reduction=12 max_load_reduction_pct=0.000362 moved_to_replicas=12 active_clone_links=1 calls=1 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=7 iteration=11 records=10 total_expected_recv=327230 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=77824 peak_demand=33015 required_capacity=1.007538 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=10 total_expected_recv=327760 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=10 total_expected_recv=327862 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76160 peak_demand=33065 required_capacity=1.009064 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=6 iteration=11 records=10 total_expected_recv=328332 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76672 peak_demand=33096 required_capacity=1.010010 -WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=10 total_expected_recv=327572 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=32904 required_capacity=1.004150 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=5 iteration=11 records=10 total_expected_recv=326790 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33096 required_capacity=1.010010 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=3 iteration=11 records=10 total_expected_recv=327420 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76800 peak_demand=33056 required_capacity=1.008789 WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=0 iteration=11 records=10 total_expected_recv=328474 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76416 peak_demand=33069 required_capacity=1.009186 - [2026-08-26 16:51:38] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 215.8 | throughput per GPU (TFLOP/s/GPU): 572.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.448511E+00 | loss scale: 1.0 | grad norm: 3.475 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | -[after training is done] datetime: 2026-08-26 16:51:38 -[rank0]:[W826 16:51:39.028960565 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0826 16:51:41.172000 276802 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 16:51:41.173000 276802 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276892 closing signal SIGTERM -W0826 16:51:41.173000 276802 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276893 closing signal SIGTERM -W0826 16:51:41.175000 276802 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276894 closing signal SIGTERM -W0826 16:51:41.176000 276802 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276895 closing signal SIGTERM -W0826 16:51:41.177000 276802 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276896 closing signal SIGTERM -W0826 16:51:41.178000 276802 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276897 closing signal SIGTERM -W0826 16:51:41.180000 276802 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276898 closing signal SIGTERM -W0826 16:51:41.181000 276802 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 276899 closing signal SIGTERM +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=1 iteration=11 records=10 total_expected_recv=327862 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76160 peak_demand=33065 required_capacity=1.009064 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=4 iteration=11 records=10 total_expected_recv=327572 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76928 peak_demand=32904 required_capacity=1.004150 +WARNING:megatron.training.drop_debug_utils:[MOE_DROP_DEBUG][iteration_summary] global_rank=2 iteration=11 records=10 total_expected_recv=327760 total_dropped_tokens=0 drop_rate_pct=0.000000 records_with_drop_pct=0.000000 peak_rank_overflow=0 total_slack_slots=76544 peak_demand=33096 required_capacity=1.010010 + [2026-08-26 18:03:46] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 216.1 | throughput per GPU (TFLOP/s/GPU): 571.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.456830E+00 | loss scale: 1.0 | grad norm: 3.508 | number of skipped iterations: 0 | number of nan iterations: 0 | peak recv tokens: 33664 | min safe capacity: 1.0273 | +[after training is done] datetime: 2026-08-26 18:03:46 +[rank0]:[W826 18:03:47.269925272 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 18:03:48.289000 449269 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 18:03:48.289000 449269 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 449359 closing signal SIGTERM +W0826 18:03:48.290000 449269 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 449360 closing signal SIGTERM +W0826 18:03:48.292000 449269 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 449361 closing signal SIGTERM +W0826 18:03:48.294000 449269 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 449362 closing signal SIGTERM +W0826 18:03:48.295000 449269 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 449363 closing signal SIGTERM +W0826 18:03:48.298000 449269 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 449364 closing signal SIGTERM +W0826 18:03:48.300000 449269 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 449365 closing signal SIGTERM +W0826 18:03:48.302000 449269 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 449366 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -6090,4 +6090,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 276802 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 449269 got signal: 15 diff --git a/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv index caf61f1bcee9cf4a58e4572a4992f1a6d78ec20a..72f4d7c610029c298da5dc6a6057dd30e1f0feb5 100644 --- a/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +++ b/ultraep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -6,3 +6,8 @@ QWEN3_30B ultraep measure 1.25 OK 117927 1 0.0 1.010101 /workspace/tierb_study_2 QWEN3_30B ultraep validate 1.0625 OK 116203 1 0.0 1.010101 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 QWEN3_30B ultraep forced 1.25 OK 117867 1 0.0 1.01001 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 DSV2_LITE_L24 ultraep measure 1.25 OK 130023 1 0.0 1.010071 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1 +DSV2_LITE_L24 ultraep validate 1.0625 OK 127987 1 0.0 1.010071 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0625/rep1 +DSV2_LITE_L24 ultraep forced 1.25 OK 129971 1 0.0 1.01001 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1 +QWEN2_57B ultraep measure 1.25 OK 97675 1 0.0 1.010101 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 +QWEN2_57B ultraep validate 1.0625 OK 96633 1 0.0 1.010132 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 +QWEN2_57B ultraep forced 1.25 OK 97677 1 0.0 1.01001 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 diff --git a/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv index ef7cd5998fddd86049d1a3a7a7faeeef3ae10361..0523284148f349a939c97ddf66072d9bcbe5b14d 100644 --- a/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +++ b/ultraep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -1,3 +1,5 @@ model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir QWEN3_235B ultraep SELECTED 1.0625 1.01004 0.0 120907 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p0625/rep1 QWEN3_30B ultraep SELECTED 1.0625 1.010101 0.0 116203 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 +DSV2_LITE_L24 ultraep SELECTED 1.0625 1.010071 0.0 127987 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0625/rep1 +QWEN2_57B ultraep SELECTED 1.0625 1.010132 0.0 96633 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 diff --git a/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.json b/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.json index 9220d6afef0dc308e624a67d08c871c17dbd0e70..3293d3273763324aa82bf6cafcdfc010241eb037 100644 --- a/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.json +++ b/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.json @@ -1,4 +1,81 @@ [ + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-3", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.0625", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "DSV2_LITE_L24", + "cuda_graph": "0", + "dense_first_layers": "1", + "drop_expected_assignments": 108527616.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "8", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 276, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "23", + "num_experts": "64", + "num_layers": "24", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 49647.0, + "peak_demand_mean": 49645.270833333336, + "peak_demand_min": 49644.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.02649873914445656, + "post_physical_rank_cv_mean": 0.020751556297178456, + "post_physical_rank_cv_min": 0.010393449619293449, + "post_physical_rank_cv_n": 276, + "post_physical_rank_imax_max": 1.01007080078125, + "post_physical_rank_imax_mean": 1.0100281941142057, + "post_physical_rank_imax_min": 1.010009765625, + "post_physical_rank_imax_n": 276, + "pre_home_rank_cv_max": 1.3897787536248405, + "pre_home_rank_cv_mean": 0.5881957299328056, + "pre_home_rank_cv_min": 0.06749209466238901, + "pre_home_rank_cv_n": 276, + "pre_home_rank_imax_max": 4.4163818359375, + "pre_home_rank_imax_mean": 2.107403446510794, + "pre_home_rank_imax_min": 1.1109822591145833, + "pre_home_rank_imax_n": 276, + "records": { + "drop_dispatch": 2208, + "drop_experts": 2208, + "drop_setup": 2208, + "ultraep_drop": 96, + "ultraep_post": 2208, + "ultraep_pre": 2208 + }, + "required_capacity_max": 1.010071, + "required_capacity_mean": 1.0100354270833332, + "required_capacity_min": 1.01001, + "required_capacity_n": 96, + "router_balancing": "seq_aux_loss", + "run_dir": "DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0625/rep1", + "seq_length": "8192", + "shared_expert_overlap": "0", + "topk": "6", + "valid_max_rank_reduction_max": 0.7712991882952357, + "valid_max_rank_reduction_mean": 0.46558143481151487, + "valid_max_rank_reduction_min": 0.09086747120332558, + "valid_max_rank_reduction_n": 276, + "zero_drop": 1 + }, { "arm": "natural", "assignment_sum_mismatches": 0, @@ -153,6 +230,83 @@ "valid_max_rank_reduction_n": 276, "zero_drop": 1 }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.0625", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "QWEN2_57B", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 31457280.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "8", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 120, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "10", + "num_experts": "64", + "num_layers": "10", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 33100.0, + "peak_demand_mean": 33096.125, + "peak_demand_min": 33024.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.026483363713168166, + "post_physical_rank_cv_mean": 0.016140850021413918, + "post_physical_rank_cv_min": 0.0023074094410037065, + "post_physical_rank_cv_n": 120, + "post_physical_rank_imax_max": 1.0101318359375, + "post_physical_rank_imax_mean": 1.0099934895833333, + "post_physical_rank_imax_min": 1.005340576171875, + "post_physical_rank_imax_n": 120, + "pre_home_rank_cv_max": 1.3881873634084585, + "pre_home_rank_cv_mean": 0.6345831857352783, + "pre_home_rank_cv_min": 0.0023074094410037065, + "pre_home_rank_cv_n": 120, + "pre_home_rank_imax_max": 3.98443603515625, + "pre_home_rank_imax_mean": 2.174882507324219, + "pre_home_rank_imax_min": 1.005340576171875, + "pre_home_rank_imax_n": 120, + "records": { + "drop_dispatch": 960, + "drop_experts": 960, + "drop_setup": 960, + "ultraep_drop": 96, + "ultraep_post": 960, + "ultraep_pre": 960 + }, + "required_capacity_max": 1.010132, + "required_capacity_mean": 1.0100135208333334, + "required_capacity_min": 1.007812, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.7465035768447175, + "valid_max_rank_reduction_mean": 0.49185739610567863, + "valid_max_rank_reduction_min": 0.0, + "valid_max_rank_reduction_n": 120, + "zero_drop": 1 + }, { "arm": "natural", "assignment_sum_mismatches": 0, @@ -307,6 +461,83 @@ "valid_max_rank_reduction_n": 120, "zero_drop": 1 }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-3", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.0625", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "QWEN3_235B", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 75497472.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "16", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 144, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "12", + "num_experts": "128", + "num_layers": "12", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 66194.0, + "peak_demand_mean": 66192.5625, + "peak_demand_min": 66192.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.026489124869087725, + "post_physical_rank_cv_mean": 0.014482237676166366, + "post_physical_rank_cv_min": 0.0044493697461502986, + "post_physical_rank_cv_n": 144, + "post_physical_rank_imax_max": 1.010040283203125, + "post_physical_rank_imax_mean": 1.010016229417589, + "post_physical_rank_imax_min": 1.010009765625, + "post_physical_rank_imax_n": 144, + "pre_home_rank_cv_max": 1.4325992435657406, + "pre_home_rank_cv_mean": 0.7968705628336425, + "pre_home_rank_cv_min": 0.3167748701963148, + "pre_home_rank_cv_n": 144, + "pre_home_rank_imax_max": 4.010467529296875, + "pre_home_rank_imax_mean": 2.4576683044433594, + "pre_home_rank_imax_min": 1.2317352294921875, + "pre_home_rank_imax_n": 144, + "records": { + "drop_dispatch": 1152, + "drop_experts": 1152, + "drop_setup": 1152, + "ultraep_drop": 96, + "ultraep_post": 1152, + "ultraep_pre": 1152 + }, + "required_capacity_max": 1.01004, + "required_capacity_mean": 1.0100184375, + "required_capacity_min": 1.01001, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p0625/rep1", + "seq_length": "8192", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.7481566031274969, + "valid_max_rank_reduction_mean": 0.5593977173710302, + "valid_max_rank_reduction_min": 0.17998587763091065, + "valid_max_rank_reduction_n": 144, + "zero_drop": 1 + }, { "arm": "natural", "assignment_sum_mismatches": 0, @@ -461,6 +692,83 @@ "valid_max_rank_reduction_n": 144, "zero_drop": 1 }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-3", + "backend": "ultraep", + "branch": "ultraep-token-drop-analysis", + "capacity": "1.0625", + "capacity_mode": "static", + "commit": "eaf829e7", + "config": "QWEN3_30B", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 150994944.0, + "drop_rate_pct_max": 0.0, + "drop_rate_pct_mean": 0.0, + "drop_rate_pct_min": 0.0, + "drop_rate_pct_n": 96, + "drop_rate_pct_overall": 0.0, + "dropped_tokens_sum_all_records": 0.0, + "duplicate_vector_mismatches": 0, + "experts_per_rank": "16", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 576, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "48", + "num_experts": "128", + "num_layers": "48", + "observed_dropped_tokens": 0.0, + "peak_demand_max": 33099.0, + "peak_demand_mean": 33097.114583333336, + "peak_demand_min": 33096.0, + "peak_demand_n": 96, + "peak_rank_overflow_max": 0.0, + "peak_rank_overflow_mean": 0.0, + "peak_rank_overflow_min": 0.0, + "peak_rank_overflow_n": 96, + "post_physical_rank_cv_max": 0.026494895756973648, + "post_physical_rank_cv_mean": 0.013874132660885264, + "post_physical_rank_cv_min": 0.00437342023426994, + "post_physical_rank_cv_n": 576, + "post_physical_rank_imax_max": 1.010101318359375, + "post_physical_rank_imax_mean": 1.0100233289930556, + "post_physical_rank_imax_min": 1.009979248046875, + "post_physical_rank_imax_n": 576, + "pre_home_rank_cv_max": 1.5851362931112496, + "pre_home_rank_cv_mean": 0.7430673394667031, + "pre_home_rank_cv_min": 0.14094636542440406, + "pre_home_rank_cv_n": 576, + "pre_home_rank_imax_max": 4.996368408203125, + "pre_home_rank_imax_mean": 2.309538629319933, + "pre_home_rank_imax_min": 1.2030029296875, + "pre_home_rank_imax_n": 576, + "records": { + "drop_dispatch": 4608, + "drop_experts": 4608, + "drop_setup": 4608, + "ultraep_drop": 96, + "ultraep_post": 4608, + "ultraep_pre": 4608 + }, + "required_capacity_max": 1.010101, + "required_capacity_mean": 1.01004371875, + "required_capacity_min": 1.01001, + "required_capacity_n": 96, + "router_balancing": "aux_loss", + "run_dir": "QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.7978512225065814, + "valid_max_rank_reduction_mean": 0.5290218009629275, + "valid_max_rank_reduction_min": 0.1604261796042618, + "valid_max_rank_reduction_n": 576, + "zero_drop": 1 + }, { "arm": "natural", "assignment_sum_mismatches": 0, diff --git a/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv b/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv index 8a99a0a0a2155ea719fe7f7e4a9f529e7d3a5fd9..b4b1dfd14be4087e080759906ff3cf8044b06bd3 100644 --- a/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv +++ b/ultraep/tierbprobe/tierbprobe_gbs8/probe_summary.tsv @@ -1,9 +1,13 @@ run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap assignment_sum_mismatches drop_expected_assignments drop_rate_pct_max drop_rate_pct_mean drop_rate_pct_min drop_rate_pct_n drop_rate_pct_overall dropped_tokens_sum_all_records duplicate_vector_mismatches global_snapshots incomplete_snapshots malformed_records observed_dropped_tokens peak_demand_max peak_demand_mean peak_demand_min peak_demand_n peak_rank_overflow_max peak_rank_overflow_mean peak_rank_overflow_min peak_rank_overflow_n post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n required_capacity_max required_capacity_mean required_capacity_min required_capacity_n valid_max_rank_reduction_max valid_max_rank_reduction_mean valid_max_rank_reduction_min valid_max_rank_reduction_n zero_drop +DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0625/rep1 natural DSV2_LITE_L24 ultraep 64 6 8 24 23 1 8192 8 static 1.0625 0 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 1e-3 0 0 108527616.0 0.0 0.0 0.0 96 0.0 0.0 0 276 0 0 0.0 49647.0 49645.270833333336 49644.0 96 0.0 0.0 0.0 96 0.02649873914445656 0.020751556297178456 0.010393449619293449 276 1.01007080078125 1.0100281941142057 1.010009765625 276 1.3897787536248405 0.5881957299328056 0.06749209466238901 276 4.4163818359375 2.107403446510794 1.1109822591145833 276 1.010071 1.0100354270833332 1.01001 96 0.7712991882952357 0.46558143481151487 0.09086747120332558 276 1 DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1 natural DSV2_LITE_L24 ultraep 64 6 8 24 23 1 8192 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 1e-3 0 0 108527616.0 0.0 0.0 0.0 96 0.0 0.0 0 276 0 0 0.0 49647.0 49645.270833333336 49644.0 96 0.0 0.0 0.0 96 0.02649873914445656 0.020751556297178456 0.010393449619293449 276 1.01007080078125 1.0100281941142057 1.010009765625 276 1.3897787536248405 0.5881957299328056 0.06749209466238901 276 4.4163818359375 2.107403446510794 1.1109822591145833 276 1.010071 1.0100354270833332 1.01001 96 0.7712991882952357 0.46558143481151487 0.09086747120332558 276 1 DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1 forced DSV2_LITE_L24 ultraep 64 6 8 24 23 1 8192 8 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 seq_aux_loss 1e-3 0 0 108527616.0 0.0 0.0 0.0 96 0.0 0.0 0 276 0 0 0.0 49644.0 49527.270833333336 49296.0 96 0.0 0.0 0.0 96 0.006673533732490181 0.003906455227664182 0.0014708009315538319 276 1.010009765625 1.0060408587617 1.00213623046875 276 0.006715970721309454 0.00392157619549587 0.0014708009315538319 276 1.0122884114583333 1.0060781580238527 1.00213623046875 276 1.01001 1.007635 1.00293 96 0.0022509848058525606 3.688638533861224e-05 0.0 276 1 +QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 natural QWEN2_57B ultraep 64 8 8 10 10 0 4096 8 static 1.0625 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 31457280.0 0.0 0.0 0.0 96 0.0 0.0 0 120 0 0 0.0 33100.0 33096.125 33024.0 96 0.0 0.0 0.0 96 0.026483363713168166 0.016140850021413918 0.0023074094410037065 120 1.0101318359375 1.0099934895833333 1.005340576171875 120 1.3881873634084585 0.6345831857352783 0.0023074094410037065 120 3.98443603515625 2.174882507324219 1.005340576171875 120 1.010132 1.0100135208333334 1.007812 96 0.7465035768447175 0.49185739610567863 0.0 120 1 QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 natural QWEN2_57B ultraep 64 8 8 10 10 0 4096 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 31457280.0 0.0 0.0 0.0 96 0.0 0.0 0 120 0 0 0.0 33099.0 33095.802083333336 33007.0 96 0.0 0.0 0.0 96 0.026494895756973648 0.016576524088454454 0.005105951983896318 120 1.010101318359375 1.010028330485026 1.010009765625 120 1.2658029842470846 0.6320477408555083 0.0989449408581061 120 3.9814453125 2.161820983886719 1.129058837890625 120 1.010101 1.0100037395833334 1.007294 96 0.7463208241353937 0.49456865803338423 0.10544098170121902 120 1 QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 forced QWEN2_57B ultraep 64 8 8 10 10 0 4096 8 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-2 0 0 31457280.0 0.0 0.0 0.0 96 0.0 0.0 0 120 0 0 0.0 33096.0 33004.5625 32800.0 96 0.0 0.0 0.0 96 0.01039906345549811 0.004650204419961747 0.002294812301292466 120 1.010009765625 1.0071016947428386 1.002899169921875 120 0.01039906345549811 0.004760489966476766 0.002294812301292466 120 1.015838623046875 1.0073875427246093 1.002899169921875 120 1.01001 1.0072193541666667 1.000977 96 0.005737975786343017 0.00028222750974011727 0.0 120 1 +QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p0625/rep1 natural QWEN3_235B ultraep 128 8 16 12 12 0 8192 8 static 1.0625 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-3 0 0 75497472.0 0.0 0.0 0.0 96 0.0 0.0 0 144 0 0 0.0 66194.0 66192.5625 66192.0 96 0.0 0.0 0.0 96 0.026489124869087725 0.014482237676166366 0.0044493697461502986 144 1.010040283203125 1.010016229417589 1.010009765625 144 1.4325992435657406 0.7968705628336425 0.3167748701963148 144 4.010467529296875 2.4576683044433594 1.2317352294921875 144 1.01004 1.0100184375 1.01001 96 0.7481566031274969 0.5593977173710302 0.17998587763091065 144 1 QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25/rep1 natural QWEN3_235B ultraep 128 8 16 12 12 0 8192 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-3 0 0 75497472.0 0.0 0.0 0.0 96 0.0 0.0 0 144 0 0 0.0 66195.0 66192.70833333333 66191.0 96 0.0 0.0 0.0 96 0.026489118276835043 0.014792490663171029 0.004489508120031435 144 1.0100555419921875 1.010018030802409 1.010009765625 144 1.3408957932113577 0.8120696136267035 0.3167748701963148 144 4.0847015380859375 2.495815806918674 1.2317352294921875 144 1.010056 1.0100206354166668 1.009995 96 0.7527335213582622 0.5673858465813182 0.17998587763091065 144 1 QWEN3_235B__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1 forced QWEN3_235B ultraep 128 8 16 12 12 0 8192 8 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-3 0 0 75497472.0 0.0 0.0 0.0 96 0.0 0.0 0 144 0 0 0.0 66192.0 65914.44791666667 65589.0 96 0.0 0.0 0.0 96 0.006838559601482533 0.003410368479189458 0.0012782366924169522 144 1.010009765625 1.0053078333536785 1.0019378662109375 144 0.006990045273933762 0.0034121784262079076 0.0012782366924169522 144 1.0104522705078125 1.0053127076890733 1.0019378662109375 144 1.01001 1.0057746770833333 1.000809 96 0.00043792754564262094 4.82445946182061e-06 0.0 144 1 +QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 natural QWEN3_30B ultraep 128 8 16 48 48 0 4096 8 static 1.0625 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-3 0 0 150994944.0 0.0 0.0 0.0 96 0.0 0.0 0 576 0 0 0.0 33099.0 33097.114583333336 33096.0 96 0.0 0.0 0.0 96 0.026494895756973648 0.013874132660885264 0.00437342023426994 576 1.010101318359375 1.0100233289930556 1.009979248046875 576 1.5851362931112496 0.7430673394667031 0.14094636542440406 576 4.996368408203125 2.309538629319933 1.2030029296875 576 1.010101 1.01004371875 1.01001 96 0.7978512225065814 0.5290218009629275 0.1604261796042618 576 1 QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 natural QWEN3_30B ultraep 128 8 16 48 48 0 4096 8 static 1.25 0 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-3 0 0 150994944.0 0.0 0.0 0.0 96 0.0 0.0 0 576 0 0 0.0 33099.0 33097.177083333336 33096.0 96 0.0 0.0 0.0 96 0.02649490454472685 0.013419989016426907 0.00397923942652517 576 1.010101318359375 1.0100219514634874 1.009979248046875 576 1.578658885795594 0.7653018905069695 0.06751632421014936 576 4.9940185546875 2.3694838417900934 1.1385498046875 576 1.010101 1.0100456041666666 1.01001 96 0.7977561047151133 0.5392296607977307 0.11292484185697438 576 1 QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 forced QWEN3_30B ultraep 128 8 16 48 48 0 4096 8 static 1.25 1 0 ultraep-token-drop-analysis eaf829e7 aux_loss 1e-3 0 0 150994944.0 0.0 0.0 0.0 96 0.0 0.0 0 576 0 0 0.0 33096.0 33084.114583333336 32979.0 96 0.0 0.0 0.0 96 0.008082507253598464 0.004779118222581316 0.0012089160228135533 576 1.010009765625 1.0073255962795682 1.00177001953125 576 0.009112333689945906 0.004932308997139817 0.0012089160228135533 576 1.01739501953125 1.0077196756998699 1.00177001953125 576 1.01001 1.0096472395833334 1.006439 96 0.007258983742276081 0.00038872034774766455 0.0 576 1 diff --git a/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv b/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv index e6af11e964c36447ca1daadd9ee2cdaeebd680fe..0b5075c025a978a2ab56048a87563d3b33b1dc7f 100644 --- a/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv +++ b/ultraep/tierbprobe/tierbprobe_gbs8/results.tsv @@ -6,3 +6,8 @@ QWEN3_30B ultraep matched 16 4096 8 1.25 none 1 OK 751.60 717.70 117927 /workspa QWEN3_30B ultraep matched 16 4096 8 1.0625 none 1 OK 786.81 717.30 116203 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 QWEN3_30B ultraep matched 16 4096 8 1.25 none+forcebal 1 OK 743.99 709.30 117867 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1 DSV2_LITE_L24 ultraep matched 8 8192 8 1.25 none 1 OK 640.76 639.20 130023 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25/rep1 +DSV2_LITE_L24 ultraep matched 8 8192 8 1.0625 none 1 OK 639.00 636.40 127987 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p0625/rep1 +DSV2_LITE_L24 ultraep matched 8 8192 8 1.25 none+forcebal 1 OK 626.46 625.10 129971 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__ultraep__matched__seq8192__gbs8__cap1p25__forcebal/rep1 +QWEN2_57B ultraep matched 8 4096 8 1.25 none 1 OK 294.59 234.00 97675 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25/rep1 +QWEN2_57B ultraep matched 8 4096 8 1.0625 none 1 OK 294.30 229.70 96633 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p0625/rep1 +QWEN2_57B ultraep matched 8 4096 8 1.25 none+forcebal 1 OK 273.43 211.70 97677 /workspace/tierb_study_20260826_144644/ultraep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__ultraep__matched__seq4096__gbs8__cap1p25__forcebal/rep1